diff options
Diffstat (limited to 'arch/sparc')
654 files changed, 25385 insertions, 20044 deletions
diff --git a/arch/sparc/Kbuild b/arch/sparc/Kbuild index 675afa285ddb..71cb3d934bf6 100644 --- a/arch/sparc/Kbuild +++ b/arch/sparc/Kbuild @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 # # core part of the sparc kernel # @@ -7,3 +8,7 @@ obj-y += mm/ obj-y += math-emu/ obj-y += net/ obj-y += crypto/ +obj-$(CONFIG_SPARC64) += vdso/ + +# for cleaning +subdir- += boot diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index a00cbd356db5..a630d373e645 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -1,6 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only config 64BIT - bool "64-bit kernel" if ARCH = "sparc" - default ARCH = "sparc64" + bool "64-bit kernel" if "$(ARCH)" = "sparc" + default "$(ARCH)" = "sparc64" help SPARC is a family of RISC microprocessors designed and marketed by Sun Microsystems, incorporated. They are very widely found in Sun @@ -12,80 +13,111 @@ config 64BIT config SPARC bool default y + select ARCH_HAS_CPU_CACHE_ALIASING + select ARCH_HAS_DMA_OPS + select ARCH_MIGHT_HAVE_PC_PARPORT if SPARC64 && PCI + select ARCH_MIGHT_HAVE_PC_SERIO select OF select OF_PROMTREE - select HAVE_IDE - select HAVE_OPROFILE + select HAVE_ASM_MODVERSIONS select HAVE_ARCH_KGDB if !SMP || SPARC64 select HAVE_ARCH_TRACEHOOK + select HAVE_ARCH_SECCOMP if SPARC64 + select HAVE_EXIT_THREAD + select HAVE_PCI select SYSCTL_EXCEPTION_TRACE - select ARCH_WANT_OPTIONAL_GPIOLIB - select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE select RTC_CLASS select RTC_DRV_M48T59 - select HAVE_DMA_ATTRS - select HAVE_DMA_API_DEBUG - select HAVE_ARCH_JUMP_LABEL - select HAVE_GENERIC_HARDIRQS + select RTC_SYSTOHC + select HAVE_ARCH_JUMP_LABEL if SPARC64 select GENERIC_IRQ_SHOW select ARCH_WANT_IPC_PARSE_VERSION - select USE_GENERIC_SMP_HELPERS if SMP select GENERIC_PCI_IOMAP - select HAVE_NMI_WATCHDOG if SPARC64 - select HAVE_BPF_JIT + select HAS_IOPORT + select HAVE_HARDLOCKUP_DETECTOR_SPARC64 if SPARC64 + select HAVE_CBPF_JIT if SPARC32 + select HAVE_EBPF_JIT if SPARC64 select HAVE_DEBUG_BUGVERBOSE select GENERIC_SMP_IDLE_THREAD - select GENERIC_CMOS_UPDATE - select GENERIC_CLOCKEVENTS - select GENERIC_STRNCPY_FROM_USER - select GENERIC_STRNLEN_USER select MODULES_USE_ELF_RELA + select PCI_SYSCALL if PCI + select PCI_MSI_ARCH_FALLBACKS if PCI_MSI select ODD_RT_SIGACTION select OLD_SIGSUSPEND + select CPU_NO_EFFICIENT_FFS + select LOCKDEP_SMALL if LOCKDEP + select NEED_DMA_MAP_STATE + select NEED_SG_DMA_LENGTH + select TRACE_IRQFLAGS_SUPPORT config SPARC32 def_bool !64BIT - select GENERIC_ATOMIC64 + select ARCH_32BIT_OFF_T + select ARCH_HAS_CPU_FINALIZE_INIT if !SMP + select ARCH_HAS_SYNC_DMA_FOR_CPU select CLZ_TAB + select DMA_DIRECT_REMAP + select GENERIC_ATOMIC64 + select GENERIC_LIB_CMPDI2 + select GENERIC_LIB_UCMPDI2 select HAVE_UID16 + select HAVE_PAGE_SIZE_4KB + select LOCK_MM_AND_FIND_VMA select OLD_SIGACTION + select ZONE_DMA config SPARC64 def_bool 64BIT - select ARCH_SUPPORTS_MSI + select ALTERNATE_USER_ADDRESS_SPACE select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_GRAPH_TRACER - select HAVE_FUNCTION_GRAPH_FP_TEST - select HAVE_FUNCTION_TRACE_MCOUNT_TEST select HAVE_KRETPROBES select HAVE_KPROBES - select HAVE_RCU_TABLE_FREE if SMP - select HAVE_MEMBLOCK - select HAVE_MEMBLOCK_NODE_MAP + select MMU_GATHER_RCU_TABLE_FREE if SMP + select MMU_GATHER_MERGE_VMAS + select MMU_GATHER_NO_FLUSH_CACHE select HAVE_ARCH_TRANSPARENT_HUGEPAGE select HAVE_DYNAMIC_FTRACE - select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_PAGE_SIZE_8KB select HAVE_SYSCALL_TRACEPOINTS + select HAVE_CONTEXT_TRACKING_USER + select HAVE_TIF_NOHZ select HAVE_DEBUG_KMEMLEAK + select IOMMU_HELPER + select SPARSE_IRQ select RTC_DRV_CMOS select RTC_DRV_BQ4802 select RTC_DRV_SUN4V select RTC_DRV_STARFIRE select HAVE_PERF_EVENTS select PERF_USE_VMALLOC - select IRQ_PREFLOW_FASTEOI select ARCH_HAVE_NMI_SAFE_CMPXCHG select HAVE_C_RECORDMCOUNT - select NO_BOOTMEM - -config ARCH_DEFCONFIG - string - default "arch/sparc/configs/sparc32_defconfig" if SPARC32 - default "arch/sparc/configs/sparc64_defconfig" if SPARC64 + select HAVE_ARCH_AUDITSYSCALL + select ARCH_SUPPORTS_ATOMIC_RMW + select ARCH_SUPPORTS_DEBUG_PAGEALLOC + select ARCH_SUPPORTS_HUGETLBFS + select HAVE_NMI + select HAVE_REGS_AND_STACK_ACCESS_API + select ARCH_USE_QUEUED_RWLOCKS + select ARCH_USE_QUEUED_SPINLOCKS + select GENERIC_TIME_VSYSCALL + select ARCH_CLOCKSOURCE_DATA + select ARCH_HAS_PTE_SPECIAL + select PCI_DOMAINS if PCI + select ARCH_HAS_GIGANTIC_PAGE + select HAVE_SOFTIRQ_ON_OWN_STACK + select HAVE_SETUP_PER_CPU_AREA + select NEED_PER_CPU_EMBED_FIRST_CHUNK + select NEED_PER_CPU_PAGE_FIRST_CHUNK + select ARCH_SUPPORTS_SCHED_SMT if SMP + select ARCH_SUPPORTS_SCHED_MC if SMP + +config ARCH_PROC_KCORE_TEXT + def_bool y -config IOMMU_HELPER - bool - default y if SPARC64 +config CPU_BIG_ENDIAN + def_bool y config STACKTRACE_SUPPORT bool @@ -95,10 +127,6 @@ config LOCKDEP_SUPPORT bool default y if SPARC64 -config HAVE_LATENCYTOP_SUPPORT - bool - default y if SPARC64 - config ARCH_HIBERNATION_POSSIBLE def_bool y if SPARC64 @@ -106,15 +134,6 @@ config AUDIT_ARCH bool default y -config HAVE_SETUP_PER_CPU_AREA - def_bool y if SPARC64 - -config NEED_PER_CPU_EMBED_FIRST_CHUNK - def_bool y if SPARC64 - -config NEED_PER_CPU_PAGE_FIRST_CHUNK - def_bool y if SPARC64 - config MMU bool default y @@ -122,49 +141,36 @@ config MMU config HIGHMEM bool default y if SPARC32 + select KMAP_LOCAL -config ZONE_DMA - bool - default y if SPARC32 - -config NEED_DMA_MAP_STATE - def_bool y - -config NEED_SG_DMA_LENGTH - def_bool y - -config GENERIC_ISA_DMA - bool - default y if SPARC32 +config PGTABLE_LEVELS + default 4 if 64BIT + default 3 -config ARCH_SUPPORTS_DEBUG_PAGEALLOC +config ARCH_SUPPORTS_UPROBES def_bool y if SPARC64 -source "init/Kconfig" - -source "kernel/Kconfig.freezer" - menu "Processor type and features" config SMP bool "Symmetric multi-processing support" - ---help--- + help This enables support for systems with more than one CPU. If you have a system with only one CPU, say N. If you have a system with more than one CPU, say Y. - If you say N here, the kernel will run on single and multiprocessor + If you say N here, the kernel will run on uni- and multiprocessor machines, but will use only one CPU of a multiprocessor machine. If you say Y here, the kernel will run on many, but not all, - singleprocessor machines. On a singleprocessor machine, the kernel + uniprocessor machines. On a uniprocessor machine, the kernel will run faster if you say N here. People using multiprocessor machines who say Y here should also say Y to "Enhanced Real Time Clock Support", below. The "Advanced Power Management" code will be disabled if you say Y here. - See also <file:Documentation/nmi_watchdog.txt> and the SMP-HOWTO - available at <http://www.tldp.org/docs.html#howto>. + See also <file:Documentation/admin-guide/lockup-watchdogs.rst> and the SMP-HOWTO + available at <https://www.tldp.org/docs.html#howto>. If you don't know what to do here, say N. @@ -172,19 +178,11 @@ config NR_CPUS int "Maximum number of CPUs" depends on SMP range 2 32 if SPARC32 - range 2 1024 if SPARC64 + range 2 4096 if SPARC64 default 32 if SPARC32 - default 64 if SPARC64 + default 4096 if SPARC64 -source kernel/Kconfig.hz - -config RWSEM_GENERIC_SPINLOCK - bool - default y if SPARC32 - -config RWSEM_XCHGADD_ALGORITHM - bool - default y if SPARC64 +source "kernel/Kconfig.hz" config GENERIC_HWEIGHT bool @@ -220,26 +218,11 @@ config EARLYFB bool "Support for early boot text console" default y depends on SPARC64 + select FONT_SUN8x16 + select FONT_SUPPORT help Say Y here to enable a faster early framebuffer boot console. -config SECCOMP - bool "Enable seccomp to safely compute untrusted bytecode" - depends on SPARC64 && PROC_FS - default y - help - This kernel feature is useful for number crunching applications - that may need to compute untrusted bytecode during their - execution. By using pipes or other transports made available to - the process as file descriptors supporting the read/write - syscalls, it's possible to isolate those applications in - their own address space using seccomp. Once seccomp is - enabled via /proc/<pid>/seccomp, it cannot be disabled - and the task is only allowed to execute a few safe syscalls - defined by each seccomp mode. - - If unsure, say Y. Only embedded should say N here. - config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" depends on SPARC64 && SMP @@ -268,28 +251,20 @@ config US3_MC config GENERIC_LOCKBREAK bool default y - depends on SPARC64 && SMP && PREEMPT + depends on SPARC64 && SMP && PREEMPTION config NUMA bool "NUMA support" depends on SPARC64 && SMP config NODES_SHIFT - int - default "4" - depends on NEED_MULTIPLE_NODES - -# Some NUMA nodes have memory ranges that span -# other nodes. Even though a pfn is valid and -# between a node's start and end pfns, it may not -# reside on that node. See memmap_init_zone() -# for details. -config NODES_SPAN_OTHER_NODES - def_bool y - depends on NEED_MULTIPLE_NODES - -config ARCH_SELECT_MEMORY_MODEL - def_bool y if SPARC64 + int "Maximum NUMA Nodes (as a power of 2)" + range 4 5 if SPARC64 + default "5" + depends on NUMA + help + Specify the maximum number of NUMA Nodes available on the target + system. Increases memory reserved to accommodate various tables. config ARCH_SPARSEMEM_ENABLE def_bool y if SPARC64 @@ -298,31 +273,22 @@ config ARCH_SPARSEMEM_ENABLE config ARCH_SPARSEMEM_DEFAULT def_bool y if SPARC64 -source "mm/Kconfig" - -if SPARC64 -source "kernel/power/Kconfig" -endif - -config SCHED_SMT - bool "SMT (Hyperthreading) scheduler support" - depends on SPARC64 && SMP - default y +config ARCH_FORCE_MAX_ORDER + int "Order of maximal physically contiguous allocations" + default "12" help - SMT scheduler support improves the CPU scheduler's decision making - when dealing with SPARC cpus at a cost of slightly increased overhead - in some places. If unsure say N here. + The kernel page allocator limits the size of maximal physically + contiguous allocations. The limit is called MAX_PAGE_ORDER and it + defines the maximal power of two of number of pages that can be + allocated as a single contiguous block. This option allows + overriding the default setting when ability to allocate very + large blocks of physically contiguous memory is required. -config SCHED_MC - bool "Multi-core scheduler support" - depends on SPARC64 && SMP - default y - help - Multi-core scheduler support improves the CPU scheduler's decision - making when dealing with multi-core CPU chips at a cost of slightly - increased overhead in some places. If unsure say N here. + Don't change if unsure. -source "kernel/Kconfig.preempt" +if SPARC64 || COMPILE_TEST +source "kernel/power/Kconfig" +endif config CMDLINE_BOOL bool "Default bootloader kernel arguments" @@ -361,7 +327,7 @@ config SERIAL_CONSOLE bool depends on SPARC32 default y - ---help--- + help If you say Y here, it will be possible to use a serial port as the system console (the system console is the device which receives all kernel messages and warnings and which allows logins in single user @@ -386,7 +352,9 @@ config SPARC_LEON depends on SPARC32 select USB_EHCI_BIG_ENDIAN_MMIO select USB_EHCI_BIG_ENDIAN_DESC - ---help--- + select USB_UHCI_BIG_ENDIAN_MMIO + select USB_UHCI_BIG_ENDIAN_DESC + help If you say Y here if you are running on a SPARC-LEON processor. The LEON processor is a synthesizable VHDL model of the SPARC-v8 standard. LEON is part of the GRLIB collection of @@ -400,7 +368,7 @@ menu "U-Boot options" config UBOOT_LOAD_ADDR hex "uImage Load Address" default 0x40004000 - ---help--- + help U-Boot kernel load address, the address in physical address space where u-boot will place the Linux kernel before booting it. This address is normally the base address of main memory + 0x4000. @@ -408,7 +376,7 @@ config UBOOT_LOAD_ADDR config UBOOT_FLASH_ADDR hex "uImage.o Load Address" default 0x00080000 - ---help--- + help Optional setting only affecting the uImage.o ELF-image used to download the uImage file to the target using a ELF-loader other than U-Boot. It may for example be used to download an uImage to FLASH with @@ -417,7 +385,7 @@ config UBOOT_FLASH_ADDR config UBOOT_ENTRY_ADDR hex "uImage Entry Address" default 0xf0004000 - ---help--- + help Do not change this unless you know what you're doing. This is hardcoded by the SPARC32 and LEON port. @@ -445,24 +413,6 @@ config SUN_LDOMS Say Y here is you want to support virtual devices via Logical Domains. -config PCI - bool "Support for PCI and PS/2 keyboard/mouse" - help - Find out whether your system includes a PCI bus. PCI is the name of - a bus system, i.e. the way the CPU talks to the other stuff inside - your box. If you say Y here, the kernel will include drivers and - infrastructure code to support PCI bus devices. - - CONFIG_PCI is needed for all JavaStation's (including MrCoffee), - CP-1200, JavaEngine-1, Corona, Red October, and Serengeti SGSC. - All of these platforms are extremely obscure, so say N if unsure. - -config PCI_DOMAINS - def_bool PCI if SPARC64 - -config PCI_SYSCALL - def_bool PCI - config PCIC_PCI bool depends on PCI && SPARC32 && !SPARC_LEON @@ -491,10 +441,6 @@ config SPARC_GRPCI2 help Say Y here to include the GRPCI2 Host Bridge Driver. -source "drivers/pci/Kconfig" - -source "drivers/pcmcia/Kconfig" - config SUN_OPENPROMFS tristate "Openprom tree appears in /proc/openprom" help @@ -508,49 +454,25 @@ config SUN_OPENPROMFS Only choose N if you know in advance that you will not need to modify OpenPROM settings on the running system. -# Makefile helper +# Makefile helpers config SPARC64_PCI bool default y depends on SPARC64 && PCI -endmenu - -menu "Executable file formats" +config SPARC64_PCI_MSI + bool + default y + depends on SPARC64_PCI && PCI_MSI -source "fs/Kconfig.binfmt" +endmenu config COMPAT bool depends on SPARC64 default y - select COMPAT_BINFMT_ELF select HAVE_UID16 select ARCH_WANT_OLD_COMPAT_IPC select COMPAT_OLD_SIGACTION -config SYSVIPC_COMPAT - bool - depends on COMPAT && SYSVIPC - default y - -config KEYS_COMPAT - def_bool y if COMPAT && KEYS - -endmenu - -source "net/Kconfig" - -source "drivers/Kconfig" - source "drivers/sbus/char/Kconfig" - -source "fs/Kconfig" - -source "arch/sparc/Kconfig.debug" - -source "security/Kconfig" - -source "crypto/Kconfig" - -source "lib/Kconfig" diff --git a/arch/sparc/Kconfig.debug b/arch/sparc/Kconfig.debug index 6db35fba79fd..d475a056a2e0 100644 --- a/arch/sparc/Kconfig.debug +++ b/arch/sparc/Kconfig.debug @@ -1,10 +1,4 @@ -menu "Kernel hacking" - -config TRACE_IRQFLAGS_SUPPORT - bool - default y - -source "lib/Kconfig.debug" +# SPDX-License-Identifier: GPL-2.0 config DEBUG_DCFLUSH bool "D-cache flush debugging" @@ -14,11 +8,20 @@ config MCOUNT bool depends on SPARC64 depends on FUNCTION_TRACER + select ARCH_WANT_FRAME_POINTERS + select FRAME_POINTER default y -config FRAME_POINTER +config HAVE_HARDLOCKUP_DETECTOR_SPARC64 bool - depends on MCOUNT - default y + depends on HAVE_NMI + select HARDLOCKUP_DETECTOR_SPARC64 + help + Sparc64 hardlockup detector is the last one developed before adding + the common infrastructure for handling hardlockup detectors. It is + always built. It does _not_ use the common command line parameters + and sysctl interface, except for /proc/sys/kernel/nmi_watchdog. -endmenu +config HARDLOCKUP_DETECTOR_SPARC64 + bool + depends on HAVE_HARDLOCKUP_DETECTOR_SPARC64 diff --git a/arch/sparc/Makefile b/arch/sparc/Makefile index 9ff423678cbc..0400078076e5 100644 --- a/arch/sparc/Makefile +++ b/arch/sparc/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 # # sparc/Makefile # @@ -8,10 +9,10 @@ # Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) # We are not yet configured - so test on arch -ifeq ($(ARCH),sparc) - KBUILD_DEFCONFIG := sparc32_defconfig -else +ifeq ($(ARCH),sparc64) KBUILD_DEFCONFIG := sparc64_defconfig +else + KBUILD_DEFCONFIG := sparc32_defconfig endif ifeq ($(CONFIG_SPARC32),y) @@ -20,11 +21,17 @@ ifeq ($(CONFIG_SPARC32),y) # CHECKFLAGS += -D__sparc__ -LDFLAGS := -m elf32_sparc +KBUILD_LDFLAGS := -m elf32_sparc export BITS := 32 UTS_MACHINE := sparc -KBUILD_CFLAGS += -m32 -mcpu=v8 -pipe -mno-fpu -fcall-used-g5 -fcall-used-g7 +# We are adding -Wa,-Av8 to KBUILD_CFLAGS to deal with a specs bug in some +# versions of gcc. Some gcc versions won't pass -Av8 to binutils when you +# give -mcpu=v8. This silently worked with older bintutils versions but +# does not any more. +KBUILD_CFLAGS += -m32 -mcpu=v8 -pipe -mno-fpu $(call cc-option,-fcall-used-g5) $(call cc-option,-fcall-used-g7) +KBUILD_CFLAGS += -Wa,-Av8 + KBUILD_AFLAGS += -m32 -Wa,-Av8 else @@ -32,13 +39,13 @@ else # sparc64 # -CHECKFLAGS += -D__sparc__ -D__sparc_v9__ -D__arch64__ -m64 -LDFLAGS := -m elf64_sparc +CHECKFLAGS += -D__sparc__ -D__sparc_v9__ -D__arch64__ +KBUILD_LDFLAGS := -m elf64_sparc export BITS := 64 UTS_MACHINE := sparc64 KBUILD_CFLAGS += -m64 -pipe -mno-fpu -mcpu=ultrasparc -mcmodel=medlow -KBUILD_CFLAGS += -ffixed-g4 -ffixed-g5 -fcall-used-g7 -Wno-sign-compare +KBUILD_CFLAGS += -ffixed-g4 -ffixed-g5 $(call cc-option,-fcall-used-g7) -Wno-sign-compare KBUILD_CFLAGS += -Wa,--undeclared-regs KBUILD_CFLAGS += $(call cc-option,-mtune=ultrasparc3) KBUILD_AFLAGS += -m64 -mcpu=ultrasparc -Wa,--undeclared-regs @@ -49,16 +56,11 @@ endif endif -head-y := arch/sparc/kernel/head_$(BITS).o - -# See arch/sparc/Kbuild for the core part of the kernel -core-y += arch/sparc/ - libs-y += arch/sparc/prom/ libs-y += arch/sparc/lib/ -drivers-$(CONFIG_PM) += arch/sparc/power/ -drivers-$(CONFIG_OPROFILE) += arch/sparc/oprofile/ +drivers-$(CONFIG_PM) += arch/sparc/power/ +drivers-$(CONFIG_VIDEO) += arch/sparc/video/ boot := arch/sparc/boot @@ -68,25 +70,24 @@ all: zImage image zImage uImage tftpboot.img vmlinux.aout: vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ -archclean: - $(Q)$(MAKE) $(clean)=$(boot) +install: + $(call cmd,install) + +archheaders: + $(Q)$(MAKE) $(build)=arch/sparc/kernel/syscalls all + +vdso-install-$(CONFIG_SPARC64) += arch/sparc/vdso/vdso64.so.dbg +vdso-install-$(CONFIG_COMPAT) += arch/sparc/vdso/vdso32.so.dbg # This is the image used for packaging KBUILD_IMAGE := $(boot)/zImage # Don't use tabs in echo arguments. -ifeq ($(ARCH),sparc) define archhelp - echo '* image - kernel image ($(boot)/image)' - echo '* zImage - stripped kernel image ($(boot)/zImage)' + echo '* vmlinux - standard SPARC kernel' + echo ' image - kernel image ($(boot)/image)' + echo '* zImage - stripped/compressed kernel image ($(boot)/zImage)' echo ' uImage - U-Boot SPARC32 Image (only for LEON)' + echo ' vmlinux.aout - a.out kernel for SPARC64' echo ' tftpboot.img - image prepared for tftp' endef -else -define archhelp - echo '* vmlinux - standard sparc64 kernel' - echo '* zImage - stripped and compressed sparc64 kernel ($(boot)/zImage)' - echo ' vmlinux.aout - a.out kernel for sparc64' - echo ' tftpboot.img - image prepared for tftp' -endef -endif diff --git a/arch/sparc/boot/.gitignore b/arch/sparc/boot/.gitignore index fc6f3986c76c..f3d8569a21d1 100644 --- a/arch/sparc/boot/.gitignore +++ b/arch/sparc/boot/.gitignore @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only btfix.S btfixupprep image diff --git a/arch/sparc/boot/Makefile b/arch/sparc/boot/Makefile index 6e63afb128d9..339c42d35089 100644 --- a/arch/sparc/boot/Makefile +++ b/arch/sparc/boot/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 # Makefile for the Sparc boot stuff. # # Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) @@ -6,7 +7,7 @@ ROOT_IMG := /usr/src/root.img ELFTOAOUT := elftoaout -hostprogs-y := piggyback +hostprogs := piggyback targets := tftpboot.img image zImage vmlinux.aout clean-files := System.map @@ -21,18 +22,18 @@ ifeq ($(CONFIG_SPARC64),y) # Actual linking -$(obj)/zImage: $(obj)/image +$(obj)/zImage: $(obj)/image FORCE $(call if_changed,gzip) - @echo ' kernel: $@ is ready' + @$(kecho) 'Kernel: $@ is ready' '(#'$(or $(KBUILD_BUILD_VERSION),`cat .version`)')' $(obj)/vmlinux.aout: vmlinux FORCE $(call if_changed,elftoaout) - @echo ' kernel: $@ is ready' + @$(kecho) 'Kernel: $@ is ready' '(#'$(or $(KBUILD_BUILD_VERSION),`cat .version`)')' else -$(obj)/zImage: $(obj)/image +$(obj)/zImage: $(obj)/image FORCE $(call if_changed,strip) - @echo ' kernel: $@ is ready' + @$(kecho) 'Kernel: $@ is ready' '(#'$(or $(KBUILD_BUILD_VERSION),`cat .version`)')' # The following lines make a readable image for U-Boot. # uImage - Binary file read by U-boot @@ -43,7 +44,7 @@ OBJCOPYFLAGS_image.bin := -S -O binary -R .note -R .comment $(obj)/image.bin: $(obj)/image FORCE $(call if_changed,objcopy) -$(obj)/image.gz: $(obj)/image.bin +$(obj)/image.gz: $(obj)/image.bin FORCE $(call if_changed,gzip) UIMAGE_LOADADDR = $(CONFIG_UBOOT_LOAD_ADDR) @@ -55,16 +56,16 @@ quiet_cmd_uimage.o = UIMAGE.O $@ -r -b binary $@ -o $@.o targets += uImage -$(obj)/uImage: $(obj)/image.gz +$(obj)/uImage: $(obj)/image.gz FORCE $(call if_changed,uimage) $(call if_changed,uimage.o) - @echo ' Image $@ is ready' + @$(kecho) 'Kernel: $@ is ready' '(#'$(or $(KBUILD_BUILD_VERSION),`cat .version`)')' endif $(obj)/image: vmlinux FORCE $(call if_changed,strip) - @echo ' kernel: $@ is ready' + @$(kecho) 'Kernel: $@ is ready' '(#'$(or $(KBUILD_BUILD_VERSION),`cat .version`)')' $(obj)/tftpboot.img: $(obj)/image $(obj)/piggyback System.map $(ROOT_IMG) FORCE $(call if_changed,elftoaout) diff --git a/arch/sparc/boot/install.sh b/arch/sparc/boot/install.sh new file mode 100755 index 000000000000..68de67c5621e --- /dev/null +++ b/arch/sparc/boot/install.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# +# This file is subject to the terms and conditions of the GNU General Public +# License. See the file "COPYING" in the main directory of this archive +# for more details. +# +# Copyright (C) 1995 by Linus Torvalds +# +# Adapted from code in arch/i386/boot/Makefile by H. Peter Anvin +# +# "make install" script for SPARC architecture +# +# Arguments: +# $1 - kernel version +# $2 - kernel image file +# $3 - kernel map file +# $4 - default install path (blank if root directory) + +set -e + +if [ -f $4/vmlinuz ]; then + mv $4/vmlinuz $4/vmlinuz.old +fi + +if [ -f $4/System.map ]; then + mv $4/System.map $4/System.old +fi + +cat $2 > $4/vmlinuz +cp $3 $4/System.map diff --git a/arch/sparc/boot/piggyback.c b/arch/sparc/boot/piggyback.c index bb7c95161d71..6d74064add0a 100644 --- a/arch/sparc/boot/piggyback.c +++ b/arch/sparc/boot/piggyback.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* Simple utility to make a single-image install kernel with initial ramdisk for Sparc tftpbooting without need to set up nfs. @@ -6,19 +7,7 @@ Pete Zaitcev <zaitcev@yahoo.com> endian fixes for cross-compiles, 2000. Copyright (C) 2011 Sam Ravnborg <sam@ravnborg.org> - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ + */ #include <dirent.h> #include <stdlib.h> @@ -165,6 +154,10 @@ static off_t get_hdrs_offset(int kernelfd, const char *filename) offset -= LOOKBACK; /* skip a.out header */ offset += AOUT_TEXT_OFFSET; + if (offset < 0) { + errno = -EINVAL; + die("Calculated a negative offset, probably elftoaout generated an invalid image. Did you use a recent elftoaout ?"); + } if (lseek(kernelfd, offset, SEEK_SET) < 0) die("lseek"); if (read(kernelfd, buffer, BUFSIZE) != BUFSIZE) diff --git a/arch/sparc/configs/sparc32_defconfig b/arch/sparc/configs/sparc32_defconfig index fb23fd6b186a..e021ecfb5a77 100644 --- a/arch/sparc/configs/sparc32_defconfig +++ b/arch/sparc/configs/sparc32_defconfig @@ -1,11 +1,9 @@ -CONFIG_EXPERIMENTAL=y CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_SYSFS_DEPRECATED_V2=y CONFIG_BLK_DEV_INITRD=y # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set -CONFIG_SLAB=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set @@ -23,14 +21,11 @@ CONFIG_IP_PNP_DHCP=y CONFIG_INET_AH=y CONFIG_INET_ESP=y CONFIG_INET_IPCOMP=y -# CONFIG_INET_LRO is not set -CONFIG_IPV6_PRIVACY=y CONFIG_INET6_AH=m CONFIG_INET6_ESP=m CONFIG_INET6_IPCOMP=m CONFIG_IPV6_TUNNEL=m CONFIG_NET_PKTGEN=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_BLK_DEV_LOOP=m CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_RAM=y @@ -71,7 +66,6 @@ CONFIG_EXT2_FS_XATTR=y CONFIG_EXT2_FS_POSIX_ACL=y CONFIG_EXT2_FS_SECURITY=y CONFIG_AUTOFS_FS=m -CONFIG_AUTOFS4_FS=m CONFIG_ISO9660_FS=m CONFIG_PROC_KCORE=y CONFIG_ROMFS_FS=m @@ -79,11 +73,9 @@ CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_RPCSEC_GSS_KRB5=m CONFIG_NLS=y -# CONFIG_ENABLE_WARN_DEPRECATED is not set CONFIG_DEBUG_KERNEL=y CONFIG_DETECT_HUNG_TASK=y # CONFIG_SCHED_DEBUG is not set -# CONFIG_RCU_CPU_STALL_DETECTOR is not set CONFIG_KGDB=y CONFIG_KGDB_TESTS=y CONFIG_CRYPTO_NULL=m @@ -100,6 +92,4 @@ CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TWOFISH=m -# CONFIG_CRYPTO_ANSI_CPRNG is not set # CONFIG_CRYPTO_HW is not set -CONFIG_LIBCRC32C=m diff --git a/arch/sparc/configs/sparc64_defconfig b/arch/sparc/configs/sparc64_defconfig index 9d8521b8c854..9f3f41246ae6 100644 --- a/arch/sparc/configs/sparc64_defconfig +++ b/arch/sparc/configs/sparc64_defconfig @@ -1,5 +1,4 @@ CONFIG_64BIT=y -CONFIG_EXPERIMENTAL=y # CONFIG_LOCALVERSION_AUTO is not set CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y @@ -7,9 +6,7 @@ CONFIG_LOG_BUF_SHIFT=18 CONFIG_BLK_DEV_INITRD=y CONFIG_PERF_EVENTS=y # CONFIG_COMPAT_BRK is not set -CONFIG_SLAB=y CONFIG_PROFILING=y -CONFIG_OPROFILE=m CONFIG_KPROBES=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y @@ -29,6 +26,7 @@ CONFIG_PCI=y CONFIG_PCI_MSI=y CONFIG_SUN_OPENPROMFS=m CONFIG_BINFMT_MISC=m +CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_XFRM_USER=m @@ -47,7 +45,6 @@ CONFIG_SYN_COOKIES=y CONFIG_INET_AH=y CONFIG_INET_ESP=y CONFIG_INET_IPCOMP=y -CONFIG_IPV6_PRIVACY=y CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y CONFIG_IPV6_OPTIMISTIC_DAD=y @@ -58,7 +55,6 @@ CONFIG_IPV6_TUNNEL=m CONFIG_VLAN_8021Q=m CONFIG_NET_PKTGEN=m CONFIG_NET_TCPPROBE=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" # CONFIG_PREVENT_FIRMWARE_BUILD is not set CONFIG_CONNECTOR=m CONFIG_BLK_DEV_LOOP=m @@ -68,14 +64,12 @@ CONFIG_CDROM_PKTCDVD=m CONFIG_CDROM_PKTCDVD_WCACHE=y CONFIG_ATA_OVER_ETH=m CONFIG_SUNVDC=m -CONFIG_IDE=y -CONFIG_BLK_DEV_IDECD=y -CONFIG_BLK_DEV_ALI15X3=y +CONFIG_ATA=y +CONFIG_PATA_ALI=y CONFIG_RAID_ATTRS=m CONFIG_SCSI=y CONFIG_BLK_DEV_SD=y CONFIG_BLK_DEV_SR=m -CONFIG_BLK_DEV_SR_VENDOR=y CONFIG_CHR_DEV_SG=m CONFIG_SCSI_MULTI_LUN=y CONFIG_SCSI_CONSTANTS=y @@ -98,9 +92,10 @@ CONFIG_NETDEVICES=y CONFIG_NET_ETHERNET=y CONFIG_MII=m CONFIG_SUNLANCE=m -CONFIG_HAPPYMEAL=m +CONFIG_HAPPYMEAL=y CONFIG_SUNGEM=m CONFIG_SUNVNET=m +CONFIG_LDMVSW=m CONFIG_NET_PCI=y CONFIG_E1000=m CONFIG_E1000E=m @@ -126,7 +121,6 @@ CONFIG_INPUT_SPARCSPKR=y # CONFIG_SERIO_SERPORT is not set CONFIG_SERIO_PCIPS2=m CONFIG_SERIO_RAW=m -# CONFIG_DEVKMEM is not set CONFIG_SERIAL_SUNSU=y CONFIG_SERIAL_SUNSU_CONSOLE=y CONFIG_SERIAL_SUNSAB=y @@ -183,7 +177,6 @@ CONFIG_HID_TOPSEED=y CONFIG_HID_THRUSTMASTER=y CONFIG_HID_ZEROPLUS=y CONFIG_USB=y -# CONFIG_USB_DEVICE_CLASS is not set CONFIG_USB_EHCI_HCD=m # CONFIG_USB_EHCI_TT_NEWSCHED is not set CONFIG_USB_OHCI_HCD=y @@ -194,27 +187,24 @@ CONFIG_EXT2_FS=y CONFIG_EXT2_FS_XATTR=y CONFIG_EXT2_FS_POSIX_ACL=y CONFIG_EXT2_FS_SECURITY=y -CONFIG_EXT3_FS=y -# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set -CONFIG_EXT3_FS_POSIX_ACL=y -CONFIG_EXT3_FS_SECURITY=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y CONFIG_PROC_KCORE=y CONFIG_TMPFS=y CONFIG_HUGETLBFS=y CONFIG_PRINTK_TIME=y -# CONFIG_ENABLE_WARN_DEPRECATED is not set CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_LOCKUP_DETECTOR=y CONFIG_DETECT_HUNG_TASK=y # CONFIG_SCHED_DEBUG is not set CONFIG_SCHEDSTATS=y -# CONFIG_RCU_CPU_STALL_DETECTOR is not set -CONFIG_SYSCTL_SYSCALL_CHECK=y CONFIG_BLK_DEV_IO_TRACE=y +CONFIG_UPROBE_EVENTS=y CONFIG_KEYS=y CONFIG_CRYPTO_NULL=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m @@ -237,6 +227,8 @@ CONFIG_CRYPTO_SEED=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m -# CONFIG_CRYPTO_ANSI_CPRNG is not set -CONFIG_CRC16=m -CONFIG_LIBCRC32C=m +CONFIG_VCC=m +CONFIG_PATA_CMD64X=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_DEVTMPFS=y diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig new file mode 100644 index 000000000000..f755da979534 --- /dev/null +++ b/arch/sparc/crypto/Kconfig @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Accelerated Cryptographic Algorithms for CPU (sparc64)" + +config CRYPTO_DES_SPARC64 + tristate "Ciphers: DES and Triple DES EDE, modes: ECB/CBC" + depends on SPARC64 + select CRYPTO_ALGAPI + select CRYPTO_LIB_DES + select CRYPTO_SKCIPHER + help + Block cipher: DES (FIPS 46-2) cipher algorithm + Block cipher: Triple DES EDE (FIPS 46-3) cipher algorithm + Length-preserving ciphers: DES with ECB and CBC modes + Length-preserving ciphers: Tripe DES EDE with ECB and CBC modes + + Architecture: sparc64 + +config CRYPTO_AES_SPARC64 + tristate "Ciphers: AES, modes: ECB, CBC, CTR" + depends on SPARC64 + select CRYPTO_SKCIPHER + help + Block ciphers: AES cipher algorithms (FIPS-197) + Length-preseving ciphers: AES with ECB, CBC, and CTR modes + + Architecture: sparc64 using crypto instructions + +config CRYPTO_CAMELLIA_SPARC64 + tristate "Ciphers: Camellia, modes: ECB, CBC" + depends on SPARC64 + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + help + Block ciphers: Camellia cipher algorithms + Length-preserving ciphers: Camellia with ECB and CBC modes + + Architecture: sparc64 + +endmenu diff --git a/arch/sparc/crypto/Makefile b/arch/sparc/crypto/Makefile index 5d469d81761f..7b4796842ddd 100644 --- a/arch/sparc/crypto/Makefile +++ b/arch/sparc/crypto/Makefile @@ -1,25 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 # # Arch-specific CryptoAPI modules. # -obj-$(CONFIG_CRYPTO_SHA1_SPARC64) += sha1-sparc64.o -obj-$(CONFIG_CRYPTO_SHA256_SPARC64) += sha256-sparc64.o -obj-$(CONFIG_CRYPTO_SHA512_SPARC64) += sha512-sparc64.o -obj-$(CONFIG_CRYPTO_MD5_SPARC64) += md5-sparc64.o - obj-$(CONFIG_CRYPTO_AES_SPARC64) += aes-sparc64.o obj-$(CONFIG_CRYPTO_DES_SPARC64) += des-sparc64.o -obj-$(CONFIG_CRYPTO_DES_SPARC64) += camellia-sparc64.o - -obj-$(CONFIG_CRYPTO_CRC32C_SPARC64) += crc32c-sparc64.o - -sha1-sparc64-y := sha1_asm.o sha1_glue.o -sha256-sparc64-y := sha256_asm.o sha256_glue.o -sha512-sparc64-y := sha512_asm.o sha512_glue.o -md5-sparc64-y := md5_asm.o md5_glue.o +obj-$(CONFIG_CRYPTO_CAMELLIA_SPARC64) += camellia-sparc64.o aes-sparc64-y := aes_asm.o aes_glue.o des-sparc64-y := des_asm.o des_glue.o camellia-sparc64-y := camellia_asm.o camellia_glue.o - -crc32c-sparc64-y := crc32c_asm.o crc32c_glue.o diff --git a/arch/sparc/crypto/aes_asm.S b/arch/sparc/crypto/aes_asm.S index 1cda8aa7cb85..f291174a72a1 100644 --- a/arch/sparc/crypto/aes_asm.S +++ b/arch/sparc/crypto/aes_asm.S @@ -1,8 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> +#include <asm/opcodes.h> #include <asm/visasm.h> -#include "opcodes.h" - #define ENCRYPT_TWO_ROUNDS(KEY_BASE, I0, I1, T0, T1) \ AES_EROUND01(KEY_BASE + 0, I0, I1, T0) \ AES_EROUND23(KEY_BASE + 2, I0, I1, T1) \ diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c index 503e6d96ad4e..359f22643b05 100644 --- a/arch/sparc/crypto/aes_glue.c +++ b/arch/sparc/crypto/aes_glue.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* Glue code for AES encryption optimized for sparc64 crypto opcodes. * * This is based largely upon arch/x86/crypto/aesni-intel_glue.c @@ -23,13 +24,13 @@ #include <linux/types.h> #include <crypto/algapi.h> #include <crypto/aes.h> +#include <crypto/internal/skcipher.h> #include <asm/fpumacro.h> +#include <asm/opcodes.h> #include <asm/pstate.h> #include <asm/elf.h> -#include "opcodes.h" - struct aes_ops { void (*encrypt)(const u64 *key, const u32 *input, u32 *output); void (*decrypt)(const u64 *key, const u32 *input, u32 *output); @@ -124,7 +125,7 @@ extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input, u64 *output, unsigned int len, u64 *iv); -struct aes_ops aes128_ops = { +static struct aes_ops aes128_ops = { .encrypt = aes_sparc64_encrypt_128, .decrypt = aes_sparc64_decrypt_128, .load_encrypt_keys = aes_sparc64_load_encrypt_keys_128, @@ -136,7 +137,7 @@ struct aes_ops aes128_ops = { .ctr_crypt = aes_sparc64_ctr_crypt_128, }; -struct aes_ops aes192_ops = { +static struct aes_ops aes192_ops = { .encrypt = aes_sparc64_encrypt_192, .decrypt = aes_sparc64_decrypt_192, .load_encrypt_keys = aes_sparc64_load_encrypt_keys_192, @@ -148,7 +149,7 @@ struct aes_ops aes192_ops = { .ctr_crypt = aes_sparc64_ctr_crypt_192, }; -struct aes_ops aes256_ops = { +static struct aes_ops aes256_ops = { .encrypt = aes_sparc64_encrypt_256, .decrypt = aes_sparc64_decrypt_256, .load_encrypt_keys = aes_sparc64_load_encrypt_keys_256, @@ -167,7 +168,6 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) { struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); - u32 *flags = &tfm->crt_flags; switch (key_len) { case AES_KEYSIZE_128: @@ -186,7 +186,6 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, break; default: - *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; return -EINVAL; } @@ -196,193 +195,170 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, return 0; } -static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + return aes_set_key(crypto_skcipher_tfm(tfm), in_key, key_len); +} + +static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst); } -static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst); } -#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) - -static int ecb_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int ecb_encrypt(struct skcipher_request *req) { - struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; int err; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; ctx->ops->load_encrypt_keys(&ctx->key[0]); - while ((nbytes = walk.nbytes)) { - unsigned int block_len = nbytes & AES_BLOCK_MASK; - - if (likely(block_len)) { - ctx->ops->ecb_encrypt(&ctx->key[0], - (const u64 *)walk.src.virt.addr, - (u64 *) walk.dst.virt.addr, - block_len); - } - nbytes &= AES_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); + while ((nbytes = walk.nbytes) != 0) { + ctx->ops->ecb_encrypt(&ctx->key[0], walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE)); + err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); } fprs_write(0); return err; } -static int ecb_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int ecb_decrypt(struct skcipher_request *req) { - struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; - u64 *key_end; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + const u64 *key_end; + struct skcipher_walk walk; + unsigned int nbytes; int err; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; ctx->ops->load_decrypt_keys(&ctx->key[0]); key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)]; - while ((nbytes = walk.nbytes)) { - unsigned int block_len = nbytes & AES_BLOCK_MASK; - - if (likely(block_len)) { - ctx->ops->ecb_decrypt(key_end, - (const u64 *) walk.src.virt.addr, - (u64 *) walk.dst.virt.addr, block_len); - } - nbytes &= AES_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); + while ((nbytes = walk.nbytes) != 0) { + ctx->ops->ecb_decrypt(key_end, walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE)); + err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); } fprs_write(0); return err; } -static int cbc_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int cbc_encrypt(struct skcipher_request *req) { - struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; int err; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; ctx->ops->load_encrypt_keys(&ctx->key[0]); - while ((nbytes = walk.nbytes)) { - unsigned int block_len = nbytes & AES_BLOCK_MASK; - - if (likely(block_len)) { - ctx->ops->cbc_encrypt(&ctx->key[0], - (const u64 *)walk.src.virt.addr, - (u64 *) walk.dst.virt.addr, - block_len, (u64 *) walk.iv); - } - nbytes &= AES_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); + while ((nbytes = walk.nbytes) != 0) { + ctx->ops->cbc_encrypt(&ctx->key[0], walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + walk.iv); + err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); } fprs_write(0); return err; } -static int cbc_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int cbc_decrypt(struct skcipher_request *req) { - struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; - u64 *key_end; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + const u64 *key_end; + struct skcipher_walk walk; + unsigned int nbytes; int err; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; ctx->ops->load_decrypt_keys(&ctx->key[0]); key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)]; - while ((nbytes = walk.nbytes)) { - unsigned int block_len = nbytes & AES_BLOCK_MASK; - - if (likely(block_len)) { - ctx->ops->cbc_decrypt(key_end, - (const u64 *) walk.src.virt.addr, - (u64 *) walk.dst.virt.addr, - block_len, (u64 *) walk.iv); - } - nbytes &= AES_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); + while ((nbytes = walk.nbytes) != 0) { + ctx->ops->cbc_decrypt(key_end, walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + walk.iv); + err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); } fprs_write(0); return err; } -static void ctr_crypt_final(struct crypto_sparc64_aes_ctx *ctx, - struct blkcipher_walk *walk) +static void ctr_crypt_final(const struct crypto_sparc64_aes_ctx *ctx, + struct skcipher_walk *walk) { u8 *ctrblk = walk->iv; u64 keystream[AES_BLOCK_SIZE / sizeof(u64)]; - u8 *src = walk->src.virt.addr; + const u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; unsigned int nbytes = walk->nbytes; ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk, keystream, AES_BLOCK_SIZE); - crypto_xor((u8 *) keystream, src, nbytes); - memcpy(dst, keystream, nbytes); + crypto_xor_cpy(dst, (u8 *) keystream, src, nbytes); crypto_inc(ctrblk, AES_BLOCK_SIZE); } -static int ctr_crypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int ctr_crypt(struct skcipher_request *req) { - struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; int err; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; ctx->ops->load_encrypt_keys(&ctx->key[0]); while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { - unsigned int block_len = nbytes & AES_BLOCK_MASK; - - if (likely(block_len)) { - ctx->ops->ctr_crypt(&ctx->key[0], - (const u64 *)walk.src.virt.addr, - (u64 *) walk.dst.virt.addr, - block_len, (u64 *) walk.iv); - } - nbytes &= AES_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); + ctx->ops->ctr_crypt(&ctx->key[0], walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + walk.iv); + err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); } if (walk.nbytes) { ctr_crypt_final(ctx, &walk); - err = blkcipher_walk_done(desc, &walk, 0); + err = skcipher_walk_done(&walk, 0); } fprs_write(0); return err; } -static struct crypto_alg algs[] = { { +static struct crypto_alg cipher_alg = { .cra_name = "aes", .cra_driver_name = "aes-sparc64", .cra_priority = SPARC_CR_OPCODE_PRIORITY, @@ -396,68 +372,57 @@ static struct crypto_alg algs[] = { { .cia_min_keysize = AES_MIN_KEY_SIZE, .cia_max_keysize = AES_MAX_KEY_SIZE, .cia_setkey = aes_set_key, - .cia_encrypt = aes_encrypt, - .cia_decrypt = aes_decrypt + .cia_encrypt = crypto_aes_encrypt, + .cia_decrypt = crypto_aes_decrypt } } -}, { - .cra_name = "ecb(aes)", - .cra_driver_name = "ecb-aes-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), - .cra_alignmask = 7, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_u = { - .blkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = aes_set_key, - .encrypt = ecb_encrypt, - .decrypt = ecb_decrypt, - }, - }, -}, { - .cra_name = "cbc(aes)", - .cra_driver_name = "cbc-aes-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), - .cra_alignmask = 7, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_u = { - .blkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = aes_set_key, - .encrypt = cbc_encrypt, - .decrypt = cbc_decrypt, - }, - }, -}, { - .cra_name = "ctr(aes)", - .cra_driver_name = "ctr-aes-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), - .cra_alignmask = 7, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_u = { - .blkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = aes_set_key, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, - }, -} }; +}; + +static struct skcipher_alg skcipher_algs[] = { + { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aes_set_key_skcipher, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aes_set_key_skcipher, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aes_set_key_skcipher, + .encrypt = ctr_crypt, + .decrypt = ctr_crypt, + .chunksize = AES_BLOCK_SIZE, + } +}; static bool __init sparc64_has_aes_opcode(void) { @@ -475,30 +440,35 @@ static bool __init sparc64_has_aes_opcode(void) static int __init aes_sparc64_mod_init(void) { - int i; - - for (i = 0; i < ARRAY_SIZE(algs); i++) - INIT_LIST_HEAD(&algs[i].cra_list); + int err; - if (sparc64_has_aes_opcode()) { - pr_info("Using sparc64 aes opcodes optimized AES implementation\n"); - return crypto_register_algs(algs, ARRAY_SIZE(algs)); + if (!sparc64_has_aes_opcode()) { + pr_info("sparc64 aes opcodes not available.\n"); + return -ENODEV; } - pr_info("sparc64 aes opcodes not available.\n"); - return -ENODEV; + pr_info("Using sparc64 aes opcodes optimized AES implementation\n"); + err = crypto_register_alg(&cipher_alg); + if (err) + return err; + err = crypto_register_skciphers(skcipher_algs, + ARRAY_SIZE(skcipher_algs)); + if (err) + crypto_unregister_alg(&cipher_alg); + return err; } static void __exit aes_sparc64_mod_fini(void) { - crypto_unregister_algs(algs, ARRAY_SIZE(algs)); + crypto_unregister_alg(&cipher_alg); + crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); } module_init(aes_sparc64_mod_init); module_exit(aes_sparc64_mod_fini); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("AES Secure Hash Algorithm, sparc64 aes opcode accelerated"); +MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, sparc64 aes opcode accelerated"); -MODULE_ALIAS("aes"); +MODULE_ALIAS_CRYPTO("aes"); #include "crop_devid.c" diff --git a/arch/sparc/crypto/camellia_asm.S b/arch/sparc/crypto/camellia_asm.S index cc39553a4e43..8471b346ef54 100644 --- a/arch/sparc/crypto/camellia_asm.S +++ b/arch/sparc/crypto/camellia_asm.S @@ -1,8 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> +#include <asm/opcodes.h> #include <asm/visasm.h> -#include "opcodes.h" - #define CAMELLIA_6ROUNDS(KEY_BASE, I0, I1) \ CAMELLIA_F(KEY_BASE + 0, I1, I0, I1) \ CAMELLIA_F(KEY_BASE + 2, I0, I1, I0) \ diff --git a/arch/sparc/crypto/camellia_glue.c b/arch/sparc/crypto/camellia_glue.c index 888f6260b4ec..e7a1e1c42b99 100644 --- a/arch/sparc/crypto/camellia_glue.c +++ b/arch/sparc/crypto/camellia_glue.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* Glue code for CAMELLIA encryption optimized for sparc64 crypto opcodes. * * Copyright (C) 2012 David S. Miller <davem@davemloft.net> @@ -11,13 +12,13 @@ #include <linux/mm.h> #include <linux/types.h> #include <crypto/algapi.h> +#include <crypto/internal/skcipher.h> #include <asm/fpumacro.h> +#include <asm/opcodes.h> #include <asm/pstate.h> #include <asm/elf.h> -#include "opcodes.h" - #define CAMELLIA_MIN_KEY_SIZE 16 #define CAMELLIA_MAX_KEY_SIZE 32 #define CAMELLIA_BLOCK_SIZE 16 @@ -37,12 +38,9 @@ static int camellia_set_key(struct crypto_tfm *tfm, const u8 *_in_key, { struct camellia_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); const u32 *in_key = (const u32 *) _in_key; - u32 *flags = &tfm->crt_flags; - if (key_len != 16 && key_len != 24 && key_len != 32) { - *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + if (key_len != 16 && key_len != 24 && key_len != 32) return -EINVAL; - } ctx->key_len = key_len; @@ -51,6 +49,12 @@ static int camellia_set_key(struct crypto_tfm *tfm, const u8 *_in_key, return 0; } +static int camellia_set_key_skcipher(struct crypto_skcipher *tfm, + const u8 *in_key, unsigned int key_len) +{ + return camellia_set_key(crypto_skcipher_tfm(tfm), in_key, key_len); +} + extern void camellia_sparc64_crypt(const u64 *key, const u32 *input, u32 *output, unsigned int key_len); @@ -80,61 +84,46 @@ typedef void ecb_crypt_op(const u64 *input, u64 *output, unsigned int len, extern ecb_crypt_op camellia_sparc64_ecb_crypt_3_grand_rounds; extern ecb_crypt_op camellia_sparc64_ecb_crypt_4_grand_rounds; -#define CAMELLIA_BLOCK_MASK (~(CAMELLIA_BLOCK_SIZE - 1)) - -static int __ecb_crypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes, bool encrypt) +static int __ecb_crypt(struct skcipher_request *req, bool encrypt) { - struct camellia_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct camellia_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; ecb_crypt_op *op; const u64 *key; + unsigned int nbytes; int err; op = camellia_sparc64_ecb_crypt_3_grand_rounds; if (ctx->key_len != 16) op = camellia_sparc64_ecb_crypt_4_grand_rounds; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; if (encrypt) key = &ctx->encrypt_key[0]; else key = &ctx->decrypt_key[0]; camellia_sparc64_load_keys(key, ctx->key_len); - while ((nbytes = walk.nbytes)) { - unsigned int block_len = nbytes & CAMELLIA_BLOCK_MASK; - - if (likely(block_len)) { - const u64 *src64; - u64 *dst64; - - src64 = (const u64 *)walk.src.virt.addr; - dst64 = (u64 *) walk.dst.virt.addr; - op(src64, dst64, block_len, key); - } - nbytes &= CAMELLIA_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); + while ((nbytes = walk.nbytes) != 0) { + op(walk.src.virt.addr, walk.dst.virt.addr, + round_down(nbytes, CAMELLIA_BLOCK_SIZE), key); + err = skcipher_walk_done(&walk, nbytes % CAMELLIA_BLOCK_SIZE); } fprs_write(0); return err; } -static int ecb_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int ecb_encrypt(struct skcipher_request *req) { - return __ecb_crypt(desc, dst, src, nbytes, true); + return __ecb_crypt(req, true); } -static int ecb_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int ecb_decrypt(struct skcipher_request *req) { - return __ecb_crypt(desc, dst, src, nbytes, false); + return __ecb_crypt(req, false); } typedef void cbc_crypt_op(const u64 *input, u64 *output, unsigned int len, @@ -145,85 +134,65 @@ extern cbc_crypt_op camellia_sparc64_cbc_encrypt_4_grand_rounds; extern cbc_crypt_op camellia_sparc64_cbc_decrypt_3_grand_rounds; extern cbc_crypt_op camellia_sparc64_cbc_decrypt_4_grand_rounds; -static int cbc_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int cbc_encrypt(struct skcipher_request *req) { - struct camellia_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct camellia_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; cbc_crypt_op *op; const u64 *key; + unsigned int nbytes; int err; op = camellia_sparc64_cbc_encrypt_3_grand_rounds; if (ctx->key_len != 16) op = camellia_sparc64_cbc_encrypt_4_grand_rounds; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; key = &ctx->encrypt_key[0]; camellia_sparc64_load_keys(key, ctx->key_len); - while ((nbytes = walk.nbytes)) { - unsigned int block_len = nbytes & CAMELLIA_BLOCK_MASK; - - if (likely(block_len)) { - const u64 *src64; - u64 *dst64; - - src64 = (const u64 *)walk.src.virt.addr; - dst64 = (u64 *) walk.dst.virt.addr; - op(src64, dst64, block_len, key, - (u64 *) walk.iv); - } - nbytes &= CAMELLIA_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); + while ((nbytes = walk.nbytes) != 0) { + op(walk.src.virt.addr, walk.dst.virt.addr, + round_down(nbytes, CAMELLIA_BLOCK_SIZE), key, walk.iv); + err = skcipher_walk_done(&walk, nbytes % CAMELLIA_BLOCK_SIZE); } fprs_write(0); return err; } -static int cbc_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int cbc_decrypt(struct skcipher_request *req) { - struct camellia_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct camellia_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; cbc_crypt_op *op; const u64 *key; + unsigned int nbytes; int err; op = camellia_sparc64_cbc_decrypt_3_grand_rounds; if (ctx->key_len != 16) op = camellia_sparc64_cbc_decrypt_4_grand_rounds; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; key = &ctx->decrypt_key[0]; camellia_sparc64_load_keys(key, ctx->key_len); - while ((nbytes = walk.nbytes)) { - unsigned int block_len = nbytes & CAMELLIA_BLOCK_MASK; - - if (likely(block_len)) { - const u64 *src64; - u64 *dst64; - - src64 = (const u64 *)walk.src.virt.addr; - dst64 = (u64 *) walk.dst.virt.addr; - op(src64, dst64, block_len, key, - (u64 *) walk.iv); - } - nbytes &= CAMELLIA_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); + while ((nbytes = walk.nbytes) != 0) { + op(walk.src.virt.addr, walk.dst.virt.addr, + round_down(nbytes, CAMELLIA_BLOCK_SIZE), key, walk.iv); + err = skcipher_walk_done(&walk, nbytes % CAMELLIA_BLOCK_SIZE); } fprs_write(0); return err; } -static struct crypto_alg algs[] = { { +static struct crypto_alg cipher_alg = { .cra_name = "camellia", .cra_driver_name = "camellia-sparc64", .cra_priority = SPARC_CR_OPCODE_PRIORITY, @@ -241,45 +210,37 @@ static struct crypto_alg algs[] = { { .cia_decrypt = camellia_decrypt } } -}, { - .cra_name = "ecb(camellia)", - .cra_driver_name = "ecb-camellia-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = CAMELLIA_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct camellia_sparc64_ctx), - .cra_alignmask = 7, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_u = { - .blkcipher = { - .min_keysize = CAMELLIA_MIN_KEY_SIZE, - .max_keysize = CAMELLIA_MAX_KEY_SIZE, - .setkey = camellia_set_key, - .encrypt = ecb_encrypt, - .decrypt = ecb_decrypt, - }, - }, -}, { - .cra_name = "cbc(camellia)", - .cra_driver_name = "cbc-camellia-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = CAMELLIA_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct camellia_sparc64_ctx), - .cra_alignmask = 7, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_u = { - .blkcipher = { - .min_keysize = CAMELLIA_MIN_KEY_SIZE, - .max_keysize = CAMELLIA_MAX_KEY_SIZE, - .setkey = camellia_set_key, - .encrypt = cbc_encrypt, - .decrypt = cbc_decrypt, - }, - }, -} +}; + +static struct skcipher_alg skcipher_algs[] = { + { + .base.cra_name = "ecb(camellia)", + .base.cra_driver_name = "ecb-camellia-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct camellia_sparc64_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .setkey = camellia_set_key_skcipher, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, { + .base.cra_name = "cbc(camellia)", + .base.cra_driver_name = "cbc-camellia-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct camellia_sparc64_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .ivsize = CAMELLIA_BLOCK_SIZE, + .setkey = camellia_set_key_skcipher, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + } }; static bool __init sparc64_has_camellia_opcode(void) @@ -298,22 +259,27 @@ static bool __init sparc64_has_camellia_opcode(void) static int __init camellia_sparc64_mod_init(void) { - int i; - - for (i = 0; i < ARRAY_SIZE(algs); i++) - INIT_LIST_HEAD(&algs[i].cra_list); + int err; - if (sparc64_has_camellia_opcode()) { - pr_info("Using sparc64 camellia opcodes optimized CAMELLIA implementation\n"); - return crypto_register_algs(algs, ARRAY_SIZE(algs)); + if (!sparc64_has_camellia_opcode()) { + pr_info("sparc64 camellia opcodes not available.\n"); + return -ENODEV; } - pr_info("sparc64 camellia opcodes not available.\n"); - return -ENODEV; + pr_info("Using sparc64 camellia opcodes optimized CAMELLIA implementation\n"); + err = crypto_register_alg(&cipher_alg); + if (err) + return err; + err = crypto_register_skciphers(skcipher_algs, + ARRAY_SIZE(skcipher_algs)); + if (err) + crypto_unregister_alg(&cipher_alg); + return err; } static void __exit camellia_sparc64_mod_fini(void) { - crypto_unregister_algs(algs, ARRAY_SIZE(algs)); + crypto_unregister_alg(&cipher_alg); + crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); } module_init(camellia_sparc64_mod_init); @@ -322,6 +288,6 @@ module_exit(camellia_sparc64_mod_fini); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Camellia Cipher Algorithm, sparc64 camellia opcode accelerated"); -MODULE_ALIAS("aes"); +MODULE_ALIAS_CRYPTO("camellia"); #include "crop_devid.c" diff --git a/arch/sparc/crypto/crc32c_asm.S b/arch/sparc/crypto/crc32c_asm.S deleted file mode 100644 index 2b1976e765b5..000000000000 --- a/arch/sparc/crypto/crc32c_asm.S +++ /dev/null @@ -1,20 +0,0 @@ -#include <linux/linkage.h> -#include <asm/visasm.h> -#include <asm/asi.h> - -#include "opcodes.h" - -ENTRY(crc32c_sparc64) - /* %o0=crc32p, %o1=data_ptr, %o2=len */ - VISEntryHalf - lda [%o0] ASI_PL, %f1 -1: ldd [%o1], %f2 - CRC32C(0,2,0) - subcc %o2, 8, %o2 - bne,pt %icc, 1b - add %o1, 0x8, %o1 - sta %f1, [%o0] ASI_PL - VISExitHalf -2: retl - nop -ENDPROC(crc32c_sparc64) diff --git a/arch/sparc/crypto/crc32c_glue.c b/arch/sparc/crypto/crc32c_glue.c deleted file mode 100644 index 5162fad912ce..000000000000 --- a/arch/sparc/crypto/crc32c_glue.c +++ /dev/null @@ -1,181 +0,0 @@ -/* Glue code for CRC32C optimized for sparc64 crypto opcodes. - * - * This is based largely upon arch/x86/crypto/crc32c-intel.c - * - * Copyright (C) 2008 Intel Corporation - * Authors: Austin Zhang <austin_zhang@linux.intel.com> - * Kent Liu <kent.liu@intel.com> - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <linux/init.h> -#include <linux/module.h> -#include <linux/string.h> -#include <linux/kernel.h> -#include <linux/crc32.h> - -#include <crypto/internal/hash.h> - -#include <asm/pstate.h> -#include <asm/elf.h> - -#include "opcodes.h" - -/* - * Setting the seed allows arbitrary accumulators and flexible XOR policy - * If your algorithm starts with ~0, then XOR with ~0 before you set - * the seed. - */ -static int crc32c_sparc64_setkey(struct crypto_shash *hash, const u8 *key, - unsigned int keylen) -{ - u32 *mctx = crypto_shash_ctx(hash); - - if (keylen != sizeof(u32)) { - crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } - *(__le32 *)mctx = le32_to_cpup((__le32 *)key); - return 0; -} - -static int crc32c_sparc64_init(struct shash_desc *desc) -{ - u32 *mctx = crypto_shash_ctx(desc->tfm); - u32 *crcp = shash_desc_ctx(desc); - - *crcp = *mctx; - - return 0; -} - -extern void crc32c_sparc64(u32 *crcp, const u64 *data, unsigned int len); - -static void crc32c_compute(u32 *crcp, const u64 *data, unsigned int len) -{ - unsigned int asm_len; - - asm_len = len & ~7U; - if (asm_len) { - crc32c_sparc64(crcp, data, asm_len); - data += asm_len / 8; - len -= asm_len; - } - if (len) - *crcp = __crc32c_le(*crcp, (const unsigned char *) data, len); -} - -static int crc32c_sparc64_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - u32 *crcp = shash_desc_ctx(desc); - - crc32c_compute(crcp, (const u64 *) data, len); - - return 0; -} - -static int __crc32c_sparc64_finup(u32 *crcp, const u8 *data, unsigned int len, - u8 *out) -{ - u32 tmp = *crcp; - - crc32c_compute(&tmp, (const u64 *) data, len); - - *(__le32 *) out = ~cpu_to_le32(tmp); - return 0; -} - -static int crc32c_sparc64_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return __crc32c_sparc64_finup(shash_desc_ctx(desc), data, len, out); -} - -static int crc32c_sparc64_final(struct shash_desc *desc, u8 *out) -{ - u32 *crcp = shash_desc_ctx(desc); - - *(__le32 *) out = ~cpu_to_le32p(crcp); - return 0; -} - -static int crc32c_sparc64_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return __crc32c_sparc64_finup(crypto_shash_ctx(desc->tfm), data, len, - out); -} - -static int crc32c_sparc64_cra_init(struct crypto_tfm *tfm) -{ - u32 *key = crypto_tfm_ctx(tfm); - - *key = ~0; - - return 0; -} - -#define CHKSUM_BLOCK_SIZE 1 -#define CHKSUM_DIGEST_SIZE 4 - -static struct shash_alg alg = { - .setkey = crc32c_sparc64_setkey, - .init = crc32c_sparc64_init, - .update = crc32c_sparc64_update, - .final = crc32c_sparc64_final, - .finup = crc32c_sparc64_finup, - .digest = crc32c_sparc64_digest, - .descsize = sizeof(u32), - .digestsize = CHKSUM_DIGEST_SIZE, - .base = { - .cra_name = "crc32c", - .cra_driver_name = "crc32c-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_blocksize = CHKSUM_BLOCK_SIZE, - .cra_ctxsize = sizeof(u32), - .cra_alignmask = 7, - .cra_module = THIS_MODULE, - .cra_init = crc32c_sparc64_cra_init, - } -}; - -static bool __init sparc64_has_crc32c_opcode(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return false; - - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); - if (!(cfr & CFR_CRC32C)) - return false; - - return true; -} - -static int __init crc32c_sparc64_mod_init(void) -{ - if (sparc64_has_crc32c_opcode()) { - pr_info("Using sparc64 crc32c opcode optimized CRC32C implementation\n"); - return crypto_register_shash(&alg); - } - pr_info("sparc64 crc32c opcode not available.\n"); - return -ENODEV; -} - -static void __exit crc32c_sparc64_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_init(crc32c_sparc64_mod_init); -module_exit(crc32c_sparc64_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("CRC32c (Castagnoli), sparc64 crc32c opcode accelerated"); - -MODULE_ALIAS("crc32c"); - -#include "crop_devid.c" diff --git a/arch/sparc/crypto/crop_devid.c b/arch/sparc/crypto/crop_devid.c index 5f5724a0ae22..93f4e0fdd38c 100644 --- a/arch/sparc/crypto/crop_devid.c +++ b/arch/sparc/crypto/crop_devid.c @@ -1,5 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/mod_devicetable.h> #include <linux/module.h> -#include <linux/of_device.h> /* This is a dummy device table linked into all of the crypto * opcode drivers. It serves to trigger the module autoloading diff --git a/arch/sparc/crypto/des_asm.S b/arch/sparc/crypto/des_asm.S index b5c8fc269b5f..d534446cbef9 100644 --- a/arch/sparc/crypto/des_asm.S +++ b/arch/sparc/crypto/des_asm.S @@ -1,8 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> +#include <asm/opcodes.h> #include <asm/visasm.h> -#include "opcodes.h" - .align 32 ENTRY(des_sparc64_key_expand) /* %o0=input_key, %o1=output_key */ diff --git a/arch/sparc/crypto/des_glue.c b/arch/sparc/crypto/des_glue.c index 3065bc61f9d3..e50ec4cd57cd 100644 --- a/arch/sparc/crypto/des_glue.c +++ b/arch/sparc/crypto/des_glue.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* Glue code for DES encryption optimized for sparc64 crypto opcodes. * * Copyright (C) 2012 David S. Miller <davem@davemloft.net> @@ -11,14 +12,14 @@ #include <linux/mm.h> #include <linux/types.h> #include <crypto/algapi.h> -#include <crypto/des.h> +#include <crypto/internal/des.h> +#include <crypto/internal/skcipher.h> #include <asm/fpumacro.h> +#include <asm/opcodes.h> #include <asm/pstate.h> #include <asm/elf.h> -#include "opcodes.h" - struct des_sparc64_ctx { u64 encrypt_expkey[DES_EXPKEY_WORDS / 2]; u64 decrypt_expkey[DES_EXPKEY_WORDS / 2]; @@ -44,19 +45,15 @@ static int des_set_key(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) { struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); - u32 *flags = &tfm->crt_flags; - u32 tmp[DES_EXPKEY_WORDS]; - int ret; + int err; /* Even though we have special instructions for key expansion, - * we call des_ekey() so that we don't have to write our own + * we call des_verify_key() so that we don't have to write our own * weak key detection code. */ - ret = des_ekey(tmp, key); - if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { - *flags |= CRYPTO_TFM_RES_WEAK_KEY; - return -EINVAL; - } + err = crypto_des_verify_key(tfm, key); + if (err) + return err; des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]); encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]); @@ -64,10 +61,16 @@ static int des_set_key(struct crypto_tfm *tfm, const u8 *key, return 0; } +static int des_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + return des_set_key(crypto_skcipher_tfm(tfm), key, keylen); +} + extern void des_sparc64_crypt(const u64 *key, const u64 *input, u64 *output); -static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +static void sparc_des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); const u64 *K = ctx->encrypt_expkey; @@ -75,7 +78,7 @@ static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); } -static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +static void sparc_des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); const u64 *K = ctx->decrypt_expkey; @@ -88,131 +91,104 @@ extern void des_sparc64_load_keys(const u64 *key); extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output, unsigned int len); -#define DES_BLOCK_MASK (~(DES_BLOCK_SIZE - 1)) - -static int __ecb_crypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes, bool encrypt) +static int __ecb_crypt(struct skcipher_request *req, bool encrypt) { - struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; int err; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; if (encrypt) des_sparc64_load_keys(&ctx->encrypt_expkey[0]); else des_sparc64_load_keys(&ctx->decrypt_expkey[0]); - while ((nbytes = walk.nbytes)) { - unsigned int block_len = nbytes & DES_BLOCK_MASK; - - if (likely(block_len)) { - des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr, - (u64 *) walk.dst.virt.addr, - block_len); - } - nbytes &= DES_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); + while ((nbytes = walk.nbytes) != 0) { + des_sparc64_ecb_crypt(walk.src.virt.addr, walk.dst.virt.addr, + round_down(nbytes, DES_BLOCK_SIZE)); + err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE); } fprs_write(0); return err; } -static int ecb_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int ecb_encrypt(struct skcipher_request *req) { - return __ecb_crypt(desc, dst, src, nbytes, true); + return __ecb_crypt(req, true); } -static int ecb_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int ecb_decrypt(struct skcipher_request *req) { - return __ecb_crypt(desc, dst, src, nbytes, false); + return __ecb_crypt(req, false); } extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output, unsigned int len, u64 *iv); -static int cbc_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output, + unsigned int len, u64 *iv); + +static int __cbc_crypt(struct skcipher_request *req, bool encrypt) { - struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; int err; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - - des_sparc64_load_keys(&ctx->encrypt_expkey[0]); - while ((nbytes = walk.nbytes)) { - unsigned int block_len = nbytes & DES_BLOCK_MASK; + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; - if (likely(block_len)) { - des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr, - (u64 *) walk.dst.virt.addr, - block_len, (u64 *) walk.iv); - } - nbytes &= DES_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); + if (encrypt) + des_sparc64_load_keys(&ctx->encrypt_expkey[0]); + else + des_sparc64_load_keys(&ctx->decrypt_expkey[0]); + while ((nbytes = walk.nbytes) != 0) { + if (encrypt) + des_sparc64_cbc_encrypt(walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, + DES_BLOCK_SIZE), + walk.iv); + else + des_sparc64_cbc_decrypt(walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, + DES_BLOCK_SIZE), + walk.iv); + err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE); } fprs_write(0); return err; } -extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output, - unsigned int len, u64 *iv); - -static int cbc_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int cbc_encrypt(struct skcipher_request *req) { - struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; - int err; - - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - - des_sparc64_load_keys(&ctx->decrypt_expkey[0]); - while ((nbytes = walk.nbytes)) { - unsigned int block_len = nbytes & DES_BLOCK_MASK; + return __cbc_crypt(req, true); +} - if (likely(block_len)) { - des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr, - (u64 *) walk.dst.virt.addr, - block_len, (u64 *) walk.iv); - } - nbytes &= DES_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); - } - fprs_write(0); - return err; +static int cbc_decrypt(struct skcipher_request *req) +{ + return __cbc_crypt(req, false); } static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) { struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); - const u32 *K = (const u32 *)key; - u32 *flags = &tfm->crt_flags; u64 k1[DES_EXPKEY_WORDS / 2]; u64 k2[DES_EXPKEY_WORDS / 2]; u64 k3[DES_EXPKEY_WORDS / 2]; + int err; - if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || - !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && - (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { - *flags |= CRYPTO_TFM_RES_WEAK_KEY; - return -EINVAL; - } + err = crypto_des3_ede_verify_key(tfm, key); + if (err) + return err; des_sparc64_key_expand((const u32 *)key, k1); key += DES_KEY_SIZE; @@ -234,10 +210,16 @@ static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key, return 0; } +static int des3_ede_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + return des3_ede_set_key(crypto_skcipher_tfm(tfm), key, keylen); +} + extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input, u64 *output); -static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +static void sparc_des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); const u64 *K = ctx->encrypt_expkey; @@ -245,7 +227,7 @@ static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); } -static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +static void sparc_des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); const u64 *K = ctx->decrypt_expkey; @@ -258,239 +240,196 @@ extern void des3_ede_sparc64_load_keys(const u64 *key); extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input, u64 *output, unsigned int len); -static int __ecb3_crypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes, bool encrypt) +static int __ecb3_crypt(struct skcipher_request *req, bool encrypt) { - struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; const u64 *K; + unsigned int nbytes; int err; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; if (encrypt) K = &ctx->encrypt_expkey[0]; else K = &ctx->decrypt_expkey[0]; des3_ede_sparc64_load_keys(K); - while ((nbytes = walk.nbytes)) { - unsigned int block_len = nbytes & DES_BLOCK_MASK; - - if (likely(block_len)) { - const u64 *src64 = (const u64 *)walk.src.virt.addr; - des3_ede_sparc64_ecb_crypt(K, src64, - (u64 *) walk.dst.virt.addr, - block_len); - } - nbytes &= DES_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); + while ((nbytes = walk.nbytes) != 0) { + des3_ede_sparc64_ecb_crypt(K, walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, DES_BLOCK_SIZE)); + err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE); } fprs_write(0); return err; } -static int ecb3_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int ecb3_encrypt(struct skcipher_request *req) { - return __ecb3_crypt(desc, dst, src, nbytes, true); + return __ecb3_crypt(req, true); } -static int ecb3_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int ecb3_decrypt(struct skcipher_request *req) { - return __ecb3_crypt(desc, dst, src, nbytes, false); + return __ecb3_crypt(req, false); } extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input, u64 *output, unsigned int len, u64 *iv); -static int cbc3_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) -{ - struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; - const u64 *K; - int err; - - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - - K = &ctx->encrypt_expkey[0]; - des3_ede_sparc64_load_keys(K); - while ((nbytes = walk.nbytes)) { - unsigned int block_len = nbytes & DES_BLOCK_MASK; - - if (likely(block_len)) { - const u64 *src64 = (const u64 *)walk.src.virt.addr; - des3_ede_sparc64_cbc_encrypt(K, src64, - (u64 *) walk.dst.virt.addr, - block_len, - (u64 *) walk.iv); - } - nbytes &= DES_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); - } - fprs_write(0); - return err; -} - extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input, u64 *output, unsigned int len, u64 *iv); -static int cbc3_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int __cbc3_crypt(struct skcipher_request *req, bool encrypt) { - struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; const u64 *K; + unsigned int nbytes; int err; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; - K = &ctx->decrypt_expkey[0]; + if (encrypt) + K = &ctx->encrypt_expkey[0]; + else + K = &ctx->decrypt_expkey[0]; des3_ede_sparc64_load_keys(K); - while ((nbytes = walk.nbytes)) { - unsigned int block_len = nbytes & DES_BLOCK_MASK; - - if (likely(block_len)) { - const u64 *src64 = (const u64 *)walk.src.virt.addr; - des3_ede_sparc64_cbc_decrypt(K, src64, - (u64 *) walk.dst.virt.addr, - block_len, - (u64 *) walk.iv); - } - nbytes &= DES_BLOCK_SIZE - 1; - err = blkcipher_walk_done(desc, &walk, nbytes); + while ((nbytes = walk.nbytes) != 0) { + if (encrypt) + des3_ede_sparc64_cbc_encrypt(K, walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, + DES_BLOCK_SIZE), + walk.iv); + else + des3_ede_sparc64_cbc_decrypt(K, walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, + DES_BLOCK_SIZE), + walk.iv); + err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE); } fprs_write(0); return err; } -static struct crypto_alg algs[] = { { - .cra_name = "des", - .cra_driver_name = "des-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct des_sparc64_ctx), - .cra_alignmask = 7, - .cra_module = THIS_MODULE, - .cra_u = { - .cipher = { - .cia_min_keysize = DES_KEY_SIZE, - .cia_max_keysize = DES_KEY_SIZE, - .cia_setkey = des_set_key, - .cia_encrypt = des_encrypt, - .cia_decrypt = des_decrypt +static int cbc3_encrypt(struct skcipher_request *req) +{ + return __cbc3_crypt(req, true); +} + +static int cbc3_decrypt(struct skcipher_request *req) +{ + return __cbc3_crypt(req, false); +} + +static struct crypto_alg cipher_algs[] = { + { + .cra_name = "des", + .cra_driver_name = "des-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct des_sparc64_ctx), + .cra_alignmask = 7, + .cra_module = THIS_MODULE, + .cra_u = { + .cipher = { + .cia_min_keysize = DES_KEY_SIZE, + .cia_max_keysize = DES_KEY_SIZE, + .cia_setkey = des_set_key, + .cia_encrypt = sparc_des_encrypt, + .cia_decrypt = sparc_des_decrypt + } } - } -}, { - .cra_name = "ecb(des)", - .cra_driver_name = "ecb-des-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct des_sparc64_ctx), - .cra_alignmask = 7, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_u = { - .blkcipher = { - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = des_set_key, - .encrypt = ecb_encrypt, - .decrypt = ecb_decrypt, - }, - }, -}, { - .cra_name = "cbc(des)", - .cra_driver_name = "cbc-des-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct des_sparc64_ctx), - .cra_alignmask = 7, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_u = { - .blkcipher = { - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = des_set_key, - .encrypt = cbc_encrypt, - .decrypt = cbc_decrypt, - }, - }, -}, { - .cra_name = "des3_ede", - .cra_driver_name = "des3_ede-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), - .cra_alignmask = 7, - .cra_module = THIS_MODULE, - .cra_u = { - .cipher = { - .cia_min_keysize = DES3_EDE_KEY_SIZE, - .cia_max_keysize = DES3_EDE_KEY_SIZE, - .cia_setkey = des3_ede_set_key, - .cia_encrypt = des3_ede_encrypt, - .cia_decrypt = des3_ede_decrypt + }, { + .cra_name = "des3_ede", + .cra_driver_name = "des3_ede-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), + .cra_alignmask = 7, + .cra_module = THIS_MODULE, + .cra_u = { + .cipher = { + .cia_min_keysize = DES3_EDE_KEY_SIZE, + .cia_max_keysize = DES3_EDE_KEY_SIZE, + .cia_setkey = des3_ede_set_key, + .cia_encrypt = sparc_des3_ede_encrypt, + .cia_decrypt = sparc_des3_ede_decrypt + } } } -}, { - .cra_name = "ecb(des3_ede)", - .cra_driver_name = "ecb-des3_ede-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), - .cra_alignmask = 7, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_u = { - .blkcipher = { - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .setkey = des3_ede_set_key, - .encrypt = ecb3_encrypt, - .decrypt = ecb3_decrypt, - }, - }, -}, { - .cra_name = "cbc(des3_ede)", - .cra_driver_name = "cbc-des3_ede-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), - .cra_alignmask = 7, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_u = { - .blkcipher = { - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .setkey = des3_ede_set_key, - .encrypt = cbc3_encrypt, - .decrypt = cbc3_decrypt, - }, - }, -} }; +}; + +static struct skcipher_alg skcipher_algs[] = { + { + .base.cra_name = "ecb(des)", + .base.cra_driver_name = "ecb-des-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct des_sparc64_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = des_set_key_skcipher, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, { + .base.cra_name = "cbc(des)", + .base.cra_driver_name = "cbc-des-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = DES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct des_sparc64_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = des_set_key_skcipher, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, { + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "ecb-des3_ede-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = des3_ede_set_key_skcipher, + .encrypt = ecb3_encrypt, + .decrypt = ecb3_decrypt, + }, { + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "cbc-des3_ede-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = des3_ede_set_key_skcipher, + .encrypt = cbc3_encrypt, + .decrypt = cbc3_decrypt, + } +}; static bool __init sparc64_has_des_opcode(void) { @@ -508,22 +447,27 @@ static bool __init sparc64_has_des_opcode(void) static int __init des_sparc64_mod_init(void) { - int i; - - for (i = 0; i < ARRAY_SIZE(algs); i++) - INIT_LIST_HEAD(&algs[i].cra_list); + int err; - if (sparc64_has_des_opcode()) { - pr_info("Using sparc64 des opcodes optimized DES implementation\n"); - return crypto_register_algs(algs, ARRAY_SIZE(algs)); + if (!sparc64_has_des_opcode()) { + pr_info("sparc64 des opcodes not available.\n"); + return -ENODEV; } - pr_info("sparc64 des opcodes not available.\n"); - return -ENODEV; + pr_info("Using sparc64 des opcodes optimized DES implementation\n"); + err = crypto_register_algs(cipher_algs, ARRAY_SIZE(cipher_algs)); + if (err) + return err; + err = crypto_register_skciphers(skcipher_algs, + ARRAY_SIZE(skcipher_algs)); + if (err) + crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs)); + return err; } static void __exit des_sparc64_mod_fini(void) { - crypto_unregister_algs(algs, ARRAY_SIZE(algs)); + crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs)); + crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); } module_init(des_sparc64_mod_init); @@ -532,6 +476,7 @@ module_exit(des_sparc64_mod_fini); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated"); -MODULE_ALIAS("des"); +MODULE_ALIAS_CRYPTO("des"); +MODULE_ALIAS_CRYPTO("des3_ede"); #include "crop_devid.c" diff --git a/arch/sparc/crypto/md5_asm.S b/arch/sparc/crypto/md5_asm.S deleted file mode 100644 index 3150404e602e..000000000000 --- a/arch/sparc/crypto/md5_asm.S +++ /dev/null @@ -1,70 +0,0 @@ -#include <linux/linkage.h> -#include <asm/visasm.h> - -#include "opcodes.h" - -ENTRY(md5_sparc64_transform) - /* %o0 = digest, %o1 = data, %o2 = rounds */ - VISEntryHalf - ld [%o0 + 0x00], %f0 - ld [%o0 + 0x04], %f1 - andcc %o1, 0x7, %g0 - ld [%o0 + 0x08], %f2 - bne,pn %xcc, 10f - ld [%o0 + 0x0c], %f3 - -1: - ldd [%o1 + 0x00], %f8 - ldd [%o1 + 0x08], %f10 - ldd [%o1 + 0x10], %f12 - ldd [%o1 + 0x18], %f14 - ldd [%o1 + 0x20], %f16 - ldd [%o1 + 0x28], %f18 - ldd [%o1 + 0x30], %f20 - ldd [%o1 + 0x38], %f22 - - MD5 - - subcc %o2, 1, %o2 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - -5: - st %f0, [%o0 + 0x00] - st %f1, [%o0 + 0x04] - st %f2, [%o0 + 0x08] - st %f3, [%o0 + 0x0c] - retl - VISExitHalf -10: - alignaddr %o1, %g0, %o1 - - ldd [%o1 + 0x00], %f10 -1: - ldd [%o1 + 0x08], %f12 - ldd [%o1 + 0x10], %f14 - ldd [%o1 + 0x18], %f16 - ldd [%o1 + 0x20], %f18 - ldd [%o1 + 0x28], %f20 - ldd [%o1 + 0x30], %f22 - ldd [%o1 + 0x38], %f24 - ldd [%o1 + 0x40], %f26 - - faligndata %f10, %f12, %f8 - faligndata %f12, %f14, %f10 - faligndata %f14, %f16, %f12 - faligndata %f16, %f18, %f14 - faligndata %f18, %f20, %f16 - faligndata %f20, %f22, %f18 - faligndata %f22, %f24, %f20 - faligndata %f24, %f26, %f22 - - MD5 - - subcc %o2, 1, %o2 - fsrc2 %f26, %f10 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - - ba,a,pt %xcc, 5b -ENDPROC(md5_sparc64_transform) diff --git a/arch/sparc/crypto/md5_glue.c b/arch/sparc/crypto/md5_glue.c deleted file mode 100644 index 09a9ea1dfb69..000000000000 --- a/arch/sparc/crypto/md5_glue.c +++ /dev/null @@ -1,190 +0,0 @@ -/* Glue code for MD5 hashing optimized for sparc64 crypto opcodes. - * - * This is based largely upon arch/x86/crypto/sha1_ssse3_glue.c - * and crypto/md5.c which are: - * - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> - * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> - * Copyright (c) Mathias Krause <minipli@googlemail.com> - * Copyright (c) Cryptoapi developers. - * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/cryptohash.h> -#include <linux/types.h> -#include <crypto/md5.h> - -#include <asm/pstate.h> -#include <asm/elf.h> - -#include "opcodes.h" - -asmlinkage void md5_sparc64_transform(u32 *digest, const char *data, - unsigned int rounds); - -static int md5_sparc64_init(struct shash_desc *desc) -{ - struct md5_state *mctx = shash_desc_ctx(desc); - - mctx->hash[0] = cpu_to_le32(0x67452301); - mctx->hash[1] = cpu_to_le32(0xefcdab89); - mctx->hash[2] = cpu_to_le32(0x98badcfe); - mctx->hash[3] = cpu_to_le32(0x10325476); - mctx->byte_count = 0; - - return 0; -} - -static void __md5_sparc64_update(struct md5_state *sctx, const u8 *data, - unsigned int len, unsigned int partial) -{ - unsigned int done = 0; - - sctx->byte_count += len; - if (partial) { - done = MD5_HMAC_BLOCK_SIZE - partial; - memcpy((u8 *)sctx->block + partial, data, done); - md5_sparc64_transform(sctx->hash, (u8 *)sctx->block, 1); - } - if (len - done >= MD5_HMAC_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / MD5_HMAC_BLOCK_SIZE; - - md5_sparc64_transform(sctx->hash, data + done, rounds); - done += rounds * MD5_HMAC_BLOCK_SIZE; - } - - memcpy(sctx->block, data + done, len - done); -} - -static int md5_sparc64_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct md5_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->byte_count % MD5_HMAC_BLOCK_SIZE; - - /* Handle the fast case right here */ - if (partial + len < MD5_HMAC_BLOCK_SIZE) { - sctx->byte_count += len; - memcpy((u8 *)sctx->block + partial, data, len); - } else - __md5_sparc64_update(sctx, data, len, partial); - - return 0; -} - -/* Add padding and return the message digest. */ -static int md5_sparc64_final(struct shash_desc *desc, u8 *out) -{ - struct md5_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - u32 *dst = (u32 *)out; - __le64 bits; - static const u8 padding[MD5_HMAC_BLOCK_SIZE] = { 0x80, }; - - bits = cpu_to_le64(sctx->byte_count << 3); - - /* Pad out to 56 mod 64 and append length */ - index = sctx->byte_count % MD5_HMAC_BLOCK_SIZE; - padlen = (index < 56) ? (56 - index) : ((MD5_HMAC_BLOCK_SIZE+56) - index); - - /* We need to fill a whole block for __md5_sparc64_update() */ - if (padlen <= 56) { - sctx->byte_count += padlen; - memcpy((u8 *)sctx->block + index, padding, padlen); - } else { - __md5_sparc64_update(sctx, padding, padlen, index); - } - __md5_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56); - - /* Store state in digest */ - for (i = 0; i < MD5_HASH_WORDS; i++) - dst[i] = sctx->hash[i]; - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int md5_sparc64_export(struct shash_desc *desc, void *out) -{ - struct md5_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - - return 0; -} - -static int md5_sparc64_import(struct shash_desc *desc, const void *in) -{ - struct md5_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - - return 0; -} - -static struct shash_alg alg = { - .digestsize = MD5_DIGEST_SIZE, - .init = md5_sparc64_init, - .update = md5_sparc64_update, - .final = md5_sparc64_final, - .export = md5_sparc64_export, - .import = md5_sparc64_import, - .descsize = sizeof(struct md5_state), - .statesize = sizeof(struct md5_state), - .base = { - .cra_name = "md5", - .cra_driver_name= "md5-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_blocksize = MD5_HMAC_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static bool __init sparc64_has_md5_opcode(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return false; - - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); - if (!(cfr & CFR_MD5)) - return false; - - return true; -} - -static int __init md5_sparc64_mod_init(void) -{ - if (sparc64_has_md5_opcode()) { - pr_info("Using sparc64 md5 opcode optimized MD5 implementation\n"); - return crypto_register_shash(&alg); - } - pr_info("sparc64 md5 opcode not available.\n"); - return -ENODEV; -} - -static void __exit md5_sparc64_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_init(md5_sparc64_mod_init); -module_exit(md5_sparc64_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("MD5 Secure Hash Algorithm, sparc64 md5 opcode accelerated"); - -MODULE_ALIAS("md5"); - -#include "crop_devid.c" diff --git a/arch/sparc/crypto/sha1_asm.S b/arch/sparc/crypto/sha1_asm.S deleted file mode 100644 index 219d10c5ae0e..000000000000 --- a/arch/sparc/crypto/sha1_asm.S +++ /dev/null @@ -1,72 +0,0 @@ -#include <linux/linkage.h> -#include <asm/visasm.h> - -#include "opcodes.h" - -ENTRY(sha1_sparc64_transform) - /* %o0 = digest, %o1 = data, %o2 = rounds */ - VISEntryHalf - ld [%o0 + 0x00], %f0 - ld [%o0 + 0x04], %f1 - ld [%o0 + 0x08], %f2 - andcc %o1, 0x7, %g0 - ld [%o0 + 0x0c], %f3 - bne,pn %xcc, 10f - ld [%o0 + 0x10], %f4 - -1: - ldd [%o1 + 0x00], %f8 - ldd [%o1 + 0x08], %f10 - ldd [%o1 + 0x10], %f12 - ldd [%o1 + 0x18], %f14 - ldd [%o1 + 0x20], %f16 - ldd [%o1 + 0x28], %f18 - ldd [%o1 + 0x30], %f20 - ldd [%o1 + 0x38], %f22 - - SHA1 - - subcc %o2, 1, %o2 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - -5: - st %f0, [%o0 + 0x00] - st %f1, [%o0 + 0x04] - st %f2, [%o0 + 0x08] - st %f3, [%o0 + 0x0c] - st %f4, [%o0 + 0x10] - retl - VISExitHalf -10: - alignaddr %o1, %g0, %o1 - - ldd [%o1 + 0x00], %f10 -1: - ldd [%o1 + 0x08], %f12 - ldd [%o1 + 0x10], %f14 - ldd [%o1 + 0x18], %f16 - ldd [%o1 + 0x20], %f18 - ldd [%o1 + 0x28], %f20 - ldd [%o1 + 0x30], %f22 - ldd [%o1 + 0x38], %f24 - ldd [%o1 + 0x40], %f26 - - faligndata %f10, %f12, %f8 - faligndata %f12, %f14, %f10 - faligndata %f14, %f16, %f12 - faligndata %f16, %f18, %f14 - faligndata %f18, %f20, %f16 - faligndata %f20, %f22, %f18 - faligndata %f22, %f24, %f20 - faligndata %f24, %f26, %f22 - - SHA1 - - subcc %o2, 1, %o2 - fsrc2 %f26, %f10 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - - ba,a,pt %xcc, 5b -ENDPROC(sha1_sparc64_transform) diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c deleted file mode 100644 index 6cd5f29e1e0d..000000000000 --- a/arch/sparc/crypto/sha1_glue.c +++ /dev/null @@ -1,185 +0,0 @@ -/* Glue code for SHA1 hashing optimized for sparc64 crypto opcodes. - * - * This is based largely upon arch/x86/crypto/sha1_ssse3_glue.c - * - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> - * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> - * Copyright (c) Mathias Krause <minipli@googlemail.com> - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/cryptohash.h> -#include <linux/types.h> -#include <crypto/sha.h> - -#include <asm/pstate.h> -#include <asm/elf.h> - -#include "opcodes.h" - -asmlinkage void sha1_sparc64_transform(u32 *digest, const char *data, - unsigned int rounds); - -static int sha1_sparc64_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha1_state){ - .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - }; - - return 0; -} - -static void __sha1_sparc64_update(struct sha1_state *sctx, const u8 *data, - unsigned int len, unsigned int partial) -{ - unsigned int done = 0; - - sctx->count += len; - if (partial) { - done = SHA1_BLOCK_SIZE - partial; - memcpy(sctx->buffer + partial, data, done); - sha1_sparc64_transform(sctx->state, sctx->buffer, 1); - } - if (len - done >= SHA1_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; - - sha1_sparc64_transform(sctx->state, data + done, rounds); - done += rounds * SHA1_BLOCK_SIZE; - } - - memcpy(sctx->buffer, data + done, len - done); -} - -static int sha1_sparc64_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - - /* Handle the fast case right here */ - if (partial + len < SHA1_BLOCK_SIZE) { - sctx->count += len; - memcpy(sctx->buffer + partial, data, len); - } else - __sha1_sparc64_update(sctx, data, len, partial); - - return 0; -} - -/* Add padding and return the message digest. */ -static int sha1_sparc64_final(struct shash_desc *desc, u8 *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be32 *dst = (__be32 *)out; - __be64 bits; - static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; - - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64 and append length */ - index = sctx->count % SHA1_BLOCK_SIZE; - padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); - - /* We need to fill a whole block for __sha1_sparc64_update() */ - if (padlen <= 56) { - sctx->count += padlen; - memcpy(sctx->buffer + index, padding, padlen); - } else { - __sha1_sparc64_update(sctx, padding, padlen, index); - } - __sha1_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56); - - /* Store state in digest */ - for (i = 0; i < 5; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int sha1_sparc64_export(struct shash_desc *desc, void *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - - return 0; -} - -static int sha1_sparc64_import(struct shash_desc *desc, const void *in) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - - return 0; -} - -static struct shash_alg alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_sparc64_init, - .update = sha1_sparc64_update, - .final = sha1_sparc64_final, - .export = sha1_sparc64_export, - .import = sha1_sparc64_import, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), - .base = { - .cra_name = "sha1", - .cra_driver_name= "sha1-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static bool __init sparc64_has_sha1_opcode(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return false; - - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); - if (!(cfr & CFR_SHA1)) - return false; - - return true; -} - -static int __init sha1_sparc64_mod_init(void) -{ - if (sparc64_has_sha1_opcode()) { - pr_info("Using sparc64 sha1 opcode optimized SHA-1 implementation\n"); - return crypto_register_shash(&alg); - } - pr_info("sparc64 sha1 opcode not available.\n"); - return -ENODEV; -} - -static void __exit sha1_sparc64_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_init(sha1_sparc64_mod_init); -module_exit(sha1_sparc64_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, sparc64 sha1 opcode accelerated"); - -MODULE_ALIAS("sha1"); - -#include "crop_devid.c" diff --git a/arch/sparc/crypto/sha256_asm.S b/arch/sparc/crypto/sha256_asm.S deleted file mode 100644 index b5f3d5826eb4..000000000000 --- a/arch/sparc/crypto/sha256_asm.S +++ /dev/null @@ -1,78 +0,0 @@ -#include <linux/linkage.h> -#include <asm/visasm.h> - -#include "opcodes.h" - -ENTRY(sha256_sparc64_transform) - /* %o0 = digest, %o1 = data, %o2 = rounds */ - VISEntryHalf - ld [%o0 + 0x00], %f0 - ld [%o0 + 0x04], %f1 - ld [%o0 + 0x08], %f2 - ld [%o0 + 0x0c], %f3 - ld [%o0 + 0x10], %f4 - ld [%o0 + 0x14], %f5 - andcc %o1, 0x7, %g0 - ld [%o0 + 0x18], %f6 - bne,pn %xcc, 10f - ld [%o0 + 0x1c], %f7 - -1: - ldd [%o1 + 0x00], %f8 - ldd [%o1 + 0x08], %f10 - ldd [%o1 + 0x10], %f12 - ldd [%o1 + 0x18], %f14 - ldd [%o1 + 0x20], %f16 - ldd [%o1 + 0x28], %f18 - ldd [%o1 + 0x30], %f20 - ldd [%o1 + 0x38], %f22 - - SHA256 - - subcc %o2, 1, %o2 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - -5: - st %f0, [%o0 + 0x00] - st %f1, [%o0 + 0x04] - st %f2, [%o0 + 0x08] - st %f3, [%o0 + 0x0c] - st %f4, [%o0 + 0x10] - st %f5, [%o0 + 0x14] - st %f6, [%o0 + 0x18] - st %f7, [%o0 + 0x1c] - retl - VISExitHalf -10: - alignaddr %o1, %g0, %o1 - - ldd [%o1 + 0x00], %f10 -1: - ldd [%o1 + 0x08], %f12 - ldd [%o1 + 0x10], %f14 - ldd [%o1 + 0x18], %f16 - ldd [%o1 + 0x20], %f18 - ldd [%o1 + 0x28], %f20 - ldd [%o1 + 0x30], %f22 - ldd [%o1 + 0x38], %f24 - ldd [%o1 + 0x40], %f26 - - faligndata %f10, %f12, %f8 - faligndata %f12, %f14, %f10 - faligndata %f14, %f16, %f12 - faligndata %f16, %f18, %f14 - faligndata %f18, %f20, %f16 - faligndata %f20, %f22, %f18 - faligndata %f22, %f24, %f20 - faligndata %f24, %f26, %f22 - - SHA256 - - subcc %o2, 1, %o2 - fsrc2 %f26, %f10 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - - ba,a,pt %xcc, 5b -ENDPROC(sha256_sparc64_transform) diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c deleted file mode 100644 index 04f555ab2680..000000000000 --- a/arch/sparc/crypto/sha256_glue.c +++ /dev/null @@ -1,243 +0,0 @@ -/* Glue code for SHA256 hashing optimized for sparc64 crypto opcodes. - * - * This is based largely upon crypto/sha256_generic.c - * - * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com> - * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> - * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> - * SHA224 Support Copyright 2007 Intel Corporation <jonathan.lynch@intel.com> - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/cryptohash.h> -#include <linux/types.h> -#include <crypto/sha.h> - -#include <asm/pstate.h> -#include <asm/elf.h> - -#include "opcodes.h" - -asmlinkage void sha256_sparc64_transform(u32 *digest, const char *data, - unsigned int rounds); - -static int sha224_sparc64_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; - - return 0; -} - -static int sha256_sparc64_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; - - return 0; -} - -static void __sha256_sparc64_update(struct sha256_state *sctx, const u8 *data, - unsigned int len, unsigned int partial) -{ - unsigned int done = 0; - - sctx->count += len; - if (partial) { - done = SHA256_BLOCK_SIZE - partial; - memcpy(sctx->buf + partial, data, done); - sha256_sparc64_transform(sctx->state, sctx->buf, 1); - } - if (len - done >= SHA256_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA256_BLOCK_SIZE; - - sha256_sparc64_transform(sctx->state, data + done, rounds); - done += rounds * SHA256_BLOCK_SIZE; - } - - memcpy(sctx->buf, data + done, len - done); -} - -static int sha256_sparc64_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; - - /* Handle the fast case right here */ - if (partial + len < SHA256_BLOCK_SIZE) { - sctx->count += len; - memcpy(sctx->buf + partial, data, len); - } else - __sha256_sparc64_update(sctx, data, len, partial); - - return 0; -} - -static int sha256_sparc64_final(struct shash_desc *desc, u8 *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be32 *dst = (__be32 *)out; - __be64 bits; - static const u8 padding[SHA256_BLOCK_SIZE] = { 0x80, }; - - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64 and append length */ - index = sctx->count % SHA256_BLOCK_SIZE; - padlen = (index < 56) ? (56 - index) : ((SHA256_BLOCK_SIZE+56) - index); - - /* We need to fill a whole block for __sha256_sparc64_update() */ - if (padlen <= 56) { - sctx->count += padlen; - memcpy(sctx->buf + index, padding, padlen); - } else { - __sha256_sparc64_update(sctx, padding, padlen, index); - } - __sha256_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56); - - /* Store state in digest */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int sha224_sparc64_final(struct shash_desc *desc, u8 *hash) -{ - u8 D[SHA256_DIGEST_SIZE]; - - sha256_sparc64_final(desc, D); - - memcpy(hash, D, SHA224_DIGEST_SIZE); - memset(D, 0, SHA256_DIGEST_SIZE); - - return 0; -} - -static int sha256_sparc64_export(struct shash_desc *desc, void *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int sha256_sparc64_import(struct shash_desc *desc, const void *in) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; -} - -static struct shash_alg sha256 = { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_sparc64_init, - .update = sha256_sparc64_update, - .final = sha256_sparc64_final, - .export = sha256_sparc64_export, - .import = sha256_sparc64_import, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name= "sha256-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static struct shash_alg sha224 = { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_sparc64_init, - .update = sha256_sparc64_update, - .final = sha224_sparc64_final, - .descsize = sizeof(struct sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name= "sha224-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static bool __init sparc64_has_sha256_opcode(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return false; - - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); - if (!(cfr & CFR_SHA256)) - return false; - - return true; -} - -static int __init sha256_sparc64_mod_init(void) -{ - if (sparc64_has_sha256_opcode()) { - int ret = crypto_register_shash(&sha224); - if (ret < 0) - return ret; - - ret = crypto_register_shash(&sha256); - if (ret < 0) { - crypto_unregister_shash(&sha224); - return ret; - } - - pr_info("Using sparc64 sha256 opcode optimized SHA-256/SHA-224 implementation\n"); - return 0; - } - pr_info("sparc64 sha256 opcode not available.\n"); - return -ENODEV; -} - -static void __exit sha256_sparc64_mod_fini(void) -{ - crypto_unregister_shash(&sha224); - crypto_unregister_shash(&sha256); -} - -module_init(sha256_sparc64_mod_init); -module_exit(sha256_sparc64_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-224 and SHA-256 Secure Hash Algorithm, sparc64 sha256 opcode accelerated"); - -MODULE_ALIAS("sha224"); -MODULE_ALIAS("sha256"); - -#include "crop_devid.c" diff --git a/arch/sparc/crypto/sha512_asm.S b/arch/sparc/crypto/sha512_asm.S deleted file mode 100644 index 54bfba713c0e..000000000000 --- a/arch/sparc/crypto/sha512_asm.S +++ /dev/null @@ -1,102 +0,0 @@ -#include <linux/linkage.h> -#include <asm/visasm.h> - -#include "opcodes.h" - -ENTRY(sha512_sparc64_transform) - /* %o0 = digest, %o1 = data, %o2 = rounds */ - VISEntry - ldd [%o0 + 0x00], %f0 - ldd [%o0 + 0x08], %f2 - ldd [%o0 + 0x10], %f4 - ldd [%o0 + 0x18], %f6 - ldd [%o0 + 0x20], %f8 - ldd [%o0 + 0x28], %f10 - andcc %o1, 0x7, %g0 - ldd [%o0 + 0x30], %f12 - bne,pn %xcc, 10f - ldd [%o0 + 0x38], %f14 - -1: - ldd [%o1 + 0x00], %f16 - ldd [%o1 + 0x08], %f18 - ldd [%o1 + 0x10], %f20 - ldd [%o1 + 0x18], %f22 - ldd [%o1 + 0x20], %f24 - ldd [%o1 + 0x28], %f26 - ldd [%o1 + 0x30], %f28 - ldd [%o1 + 0x38], %f30 - ldd [%o1 + 0x40], %f32 - ldd [%o1 + 0x48], %f34 - ldd [%o1 + 0x50], %f36 - ldd [%o1 + 0x58], %f38 - ldd [%o1 + 0x60], %f40 - ldd [%o1 + 0x68], %f42 - ldd [%o1 + 0x70], %f44 - ldd [%o1 + 0x78], %f46 - - SHA512 - - subcc %o2, 1, %o2 - bne,pt %xcc, 1b - add %o1, 0x80, %o1 - -5: - std %f0, [%o0 + 0x00] - std %f2, [%o0 + 0x08] - std %f4, [%o0 + 0x10] - std %f6, [%o0 + 0x18] - std %f8, [%o0 + 0x20] - std %f10, [%o0 + 0x28] - std %f12, [%o0 + 0x30] - std %f14, [%o0 + 0x38] - retl - VISExit -10: - alignaddr %o1, %g0, %o1 - - ldd [%o1 + 0x00], %f18 -1: - ldd [%o1 + 0x08], %f20 - ldd [%o1 + 0x10], %f22 - ldd [%o1 + 0x18], %f24 - ldd [%o1 + 0x20], %f26 - ldd [%o1 + 0x28], %f28 - ldd [%o1 + 0x30], %f30 - ldd [%o1 + 0x38], %f32 - ldd [%o1 + 0x40], %f34 - ldd [%o1 + 0x48], %f36 - ldd [%o1 + 0x50], %f38 - ldd [%o1 + 0x58], %f40 - ldd [%o1 + 0x60], %f42 - ldd [%o1 + 0x68], %f44 - ldd [%o1 + 0x70], %f46 - ldd [%o1 + 0x78], %f48 - ldd [%o1 + 0x80], %f50 - - faligndata %f18, %f20, %f16 - faligndata %f20, %f22, %f18 - faligndata %f22, %f24, %f20 - faligndata %f24, %f26, %f22 - faligndata %f26, %f28, %f24 - faligndata %f28, %f30, %f26 - faligndata %f30, %f32, %f28 - faligndata %f32, %f34, %f30 - faligndata %f34, %f36, %f32 - faligndata %f36, %f38, %f34 - faligndata %f38, %f40, %f36 - faligndata %f40, %f42, %f38 - faligndata %f42, %f44, %f40 - faligndata %f44, %f46, %f42 - faligndata %f46, %f48, %f44 - faligndata %f48, %f50, %f46 - - SHA512 - - subcc %o2, 1, %o2 - fsrc2 %f50, %f18 - bne,pt %xcc, 1b - add %o1, 0x80, %o1 - - ba,a,pt %xcc, 5b -ENDPROC(sha512_sparc64_transform) diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c deleted file mode 100644 index f04d1994d19a..000000000000 --- a/arch/sparc/crypto/sha512_glue.c +++ /dev/null @@ -1,228 +0,0 @@ -/* Glue code for SHA512 hashing optimized for sparc64 crypto opcodes. - * - * This is based largely upon crypto/sha512_generic.c - * - * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com> - * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> - * Copyright (c) 2003 Kyle McMartin <kyle@debian.org> - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/cryptohash.h> -#include <linux/types.h> -#include <crypto/sha.h> - -#include <asm/pstate.h> -#include <asm/elf.h> - -#include "opcodes.h" - -asmlinkage void sha512_sparc64_transform(u64 *digest, const char *data, - unsigned int rounds); - -static int sha512_sparc64_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA512_H0; - sctx->state[1] = SHA512_H1; - sctx->state[2] = SHA512_H2; - sctx->state[3] = SHA512_H3; - sctx->state[4] = SHA512_H4; - sctx->state[5] = SHA512_H5; - sctx->state[6] = SHA512_H6; - sctx->state[7] = SHA512_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - -static int sha384_sparc64_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA384_H0; - sctx->state[1] = SHA384_H1; - sctx->state[2] = SHA384_H2; - sctx->state[3] = SHA384_H3; - sctx->state[4] = SHA384_H4; - sctx->state[5] = SHA384_H5; - sctx->state[6] = SHA384_H6; - sctx->state[7] = SHA384_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - -static void __sha512_sparc64_update(struct sha512_state *sctx, const u8 *data, - unsigned int len, unsigned int partial) -{ - unsigned int done = 0; - - if ((sctx->count[0] += len) < len) - sctx->count[1]++; - if (partial) { - done = SHA512_BLOCK_SIZE - partial; - memcpy(sctx->buf + partial, data, done); - sha512_sparc64_transform(sctx->state, sctx->buf, 1); - } - if (len - done >= SHA512_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA512_BLOCK_SIZE; - - sha512_sparc64_transform(sctx->state, data + done, rounds); - done += rounds * SHA512_BLOCK_SIZE; - } - - memcpy(sctx->buf, data + done, len - done); -} - -static int sha512_sparc64_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; - - /* Handle the fast case right here */ - if (partial + len < SHA512_BLOCK_SIZE) { - if ((sctx->count[0] += len) < len) - sctx->count[1]++; - memcpy(sctx->buf + partial, data, len); - } else - __sha512_sparc64_update(sctx, data, len, partial); - - return 0; -} - -static int sha512_sparc64_final(struct shash_desc *desc, u8 *out) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be64 *dst = (__be64 *)out; - __be64 bits[2]; - static const u8 padding[SHA512_BLOCK_SIZE] = { 0x80, }; - - /* Save number of bits */ - bits[1] = cpu_to_be64(sctx->count[0] << 3); - bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); - - /* Pad out to 112 mod 128 and append length */ - index = sctx->count[0] % SHA512_BLOCK_SIZE; - padlen = (index < 112) ? (112 - index) : ((SHA512_BLOCK_SIZE+112) - index); - - /* We need to fill a whole block for __sha512_sparc64_update() */ - if (padlen <= 112) { - if ((sctx->count[0] += padlen) < padlen) - sctx->count[1]++; - memcpy(sctx->buf + index, padding, padlen); - } else { - __sha512_sparc64_update(sctx, padding, padlen, index); - } - __sha512_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 112); - - /* Store state in digest */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_be64(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int sha384_sparc64_final(struct shash_desc *desc, u8 *hash) -{ - u8 D[64]; - - sha512_sparc64_final(desc, D); - - memcpy(hash, D, 48); - memset(D, 0, 64); - - return 0; -} - -static struct shash_alg sha512 = { - .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_sparc64_init, - .update = sha512_sparc64_update, - .final = sha512_sparc64_final, - .descsize = sizeof(struct sha512_state), - .base = { - .cra_name = "sha512", - .cra_driver_name= "sha512-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static struct shash_alg sha384 = { - .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_sparc64_init, - .update = sha512_sparc64_update, - .final = sha384_sparc64_final, - .descsize = sizeof(struct sha512_state), - .base = { - .cra_name = "sha384", - .cra_driver_name= "sha384-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static bool __init sparc64_has_sha512_opcode(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return false; - - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); - if (!(cfr & CFR_SHA512)) - return false; - - return true; -} - -static int __init sha512_sparc64_mod_init(void) -{ - if (sparc64_has_sha512_opcode()) { - int ret = crypto_register_shash(&sha384); - if (ret < 0) - return ret; - - ret = crypto_register_shash(&sha512); - if (ret < 0) { - crypto_unregister_shash(&sha384); - return ret; - } - - pr_info("Using sparc64 sha512 opcode optimized SHA-512/SHA-384 implementation\n"); - return 0; - } - pr_info("sparc64 sha512 opcode not available.\n"); - return -ENODEV; -} - -static void __exit sha512_sparc64_mod_fini(void) -{ - crypto_unregister_shash(&sha384); - crypto_unregister_shash(&sha512); -} - -module_init(sha512_sparc64_mod_init); -module_exit(sha512_sparc64_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-384 and SHA-512 Secure Hash Algorithm, sparc64 sha512 opcode accelerated"); - -MODULE_ALIAS("sha384"); -MODULE_ALIAS("sha512"); - -#include "crop_devid.c" diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild index 7e4a97fbded4..17ee8a273aa6 100644 --- a/arch/sparc/include/asm/Kbuild +++ b/arch/sparc/include/asm/Kbuild @@ -1,18 +1,7 @@ -# User exported sparc header files - - -generic-y += clkdev.h -generic-y += cputime.h -generic-y += div64.h -generic-y += emergency-restart.h -generic-y += exec.h -generic-y += linkage.h -generic-y += local64.h -generic-y += mutex.h -generic-y += irq_regs.h -generic-y += local.h -generic-y += module.h -generic-y += serial.h -generic-y += trace_clock.h -generic-y += types.h -generic-y += word-at-a-time.h +# SPDX-License-Identifier: GPL-2.0 +generated-y += syscall_table_32.h +generated-y += syscall_table_64.h +generic-y += agp.h +generic-y += kvm_para.h +generic-y += mcs_spinlock.h +generic-y += text-patching.h diff --git a/arch/sparc/include/asm/adi.h b/arch/sparc/include/asm/adi.h new file mode 100644 index 000000000000..acad0d04e4c6 --- /dev/null +++ b/arch/sparc/include/asm/adi.h @@ -0,0 +1,6 @@ +#ifndef ___ASM_SPARC_ADI_H +#define ___ASM_SPARC_ADI_H +#if defined(__sparc__) && defined(__arch64__) +#include <asm/adi_64.h> +#endif +#endif diff --git a/arch/sparc/include/asm/adi_64.h b/arch/sparc/include/asm/adi_64.h new file mode 100644 index 000000000000..0c066fdab696 --- /dev/null +++ b/arch/sparc/include/asm/adi_64.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* adi_64.h: ADI related data structures + * + * Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. + * Author: Khalid Aziz (khalid.aziz@oracle.com) + */ +#ifndef __ASM_SPARC64_ADI_H +#define __ASM_SPARC64_ADI_H + +#include <linux/types.h> + +#ifndef __ASSEMBLER__ + +struct adi_caps { + __u64 blksz; + __u64 nbits; + __u64 ue_on_adi; +}; + +struct adi_config { + bool enabled; + struct adi_caps caps; +}; + +extern struct adi_config adi_state; + +extern void mdesc_adi_init(void); + +static inline bool adi_capable(void) +{ + return adi_state.enabled; +} + +static inline unsigned long adi_blksize(void) +{ + return adi_state.caps.blksz; +} + +static inline unsigned long adi_nbits(void) +{ + return adi_state.caps.nbits; +} + +#endif /* __ASSEMBLER__ */ + +#endif /* !(__ASM_SPARC64_ADI_H) */ diff --git a/arch/sparc/include/asm/agp.h b/arch/sparc/include/asm/agp.h deleted file mode 100644 index 70f52c1661bc..000000000000 --- a/arch/sparc/include/asm/agp.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef AGP_H -#define AGP_H 1 - -/* dummy for now */ - -#define map_page_into_agp(page) -#define unmap_page_from_agp(page) -#define flush_agp_cache() mb() - -/* GATT allocation. Returns/accepts GATT kernel virtual address. */ -#define alloc_gatt_pages(order) \ - ((char *)__get_free_pages(GFP_KERNEL, (order))) -#define free_gatt_pages(table, order) \ - free_pages((unsigned long)(table), (order)) - -#endif diff --git a/arch/sparc/include/asm/apb.h b/arch/sparc/include/asm/apb.h index 8f3b57db810f..b1dfb1a99f07 100644 --- a/arch/sparc/include/asm/apb.h +++ b/arch/sparc/include/asm/apb.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * apb.h: Advanced PCI Bridge Configuration Registers and Bits * diff --git a/arch/sparc/include/asm/asm-prototypes.h b/arch/sparc/include/asm/asm-prototypes.h new file mode 100644 index 000000000000..08810808ca6d --- /dev/null +++ b/arch/sparc/include/asm/asm-prototypes.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#include <linux/atomic.h> +#include <linux/uaccess.h> + +#include <asm/atomic.h> +#include <asm/cacheflush.h> +#include <asm/checksum.h> +#include <asm/delay.h> +#include <asm/ftrace.h> +#include <asm/oplib.h> +#include <asm/pgtable.h> +#include <asm/trap_block.h> +#include <asm/xor.h> + +void *__memscan_zero(void *, size_t); +void *__memscan_generic(void *, int, size_t); +void *__bzero(void *, size_t); +void VISenter(void); /* Dummy prototype to supress warning */ +#undef memcpy +#undef memset +void *memcpy(void *dest, const void *src, size_t n); +void *memset(void *s, int c, size_t n); +typedef int TItype __attribute__((mode(TI))); +TItype __multi3(TItype a, TItype b); + +s64 __ashldi3(s64, int); +s64 __lshrdi3(s64, int); +s64 __ashrdi3(s64, int); diff --git a/arch/sparc/include/asm/asm.h b/arch/sparc/include/asm/asm.h index e8e1d94b4cc9..eaed0117a838 100644 --- a/arch/sparc/include/asm/asm.h +++ b/arch/sparc/include/asm/asm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_ASM_H #define _SPARC_ASM_H diff --git a/arch/sparc/include/asm/asmmacro.h b/arch/sparc/include/asm/asmmacro.h index a0e28ef02558..49aaf6f3bc55 100644 --- a/arch/sparc/include/asm/asmmacro.h +++ b/arch/sparc/include/asm/asmmacro.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* asmmacro.h: Assembler macros. * * Copyright (C) 1996 David S. Miller (davem@caipfs.rutgers.edu) diff --git a/arch/sparc/include/asm/atomic.h b/arch/sparc/include/asm/atomic.h index 8ff83d8cc33f..425151cc4808 100644 --- a/arch/sparc/include/asm/atomic.h +++ b/arch/sparc/include/asm/atomic.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_ATOMIC_H #define ___ASM_SPARC_ATOMIC_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h index 905832aa9e9e..60ce2fe57fcd 100644 --- a/arch/sparc/include/asm/atomic_32.h +++ b/arch/sparc/include/asm/atomic_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* atomic.h: These still suck, but the I-cache hit rate is higher. * * Copyright (C) 1996 David S. Miller (davem@davemloft.net) @@ -14,48 +15,47 @@ #include <linux/types.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> #include <asm-generic/atomic64.h> +int arch_atomic_add_return(int, atomic_t *); +#define arch_atomic_add_return arch_atomic_add_return -#define ATOMIC_INIT(i) { (i) } +int arch_atomic_fetch_add(int, atomic_t *); +#define arch_atomic_fetch_add arch_atomic_fetch_add -extern int __atomic_add_return(int, atomic_t *); -extern int atomic_cmpxchg(atomic_t *, int, int); -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -extern int __atomic_add_unless(atomic_t *, int, int); -extern void atomic_set(atomic_t *, int); +int arch_atomic_fetch_and(int, atomic_t *); +#define arch_atomic_fetch_and arch_atomic_fetch_and -#define atomic_read(v) (*(volatile int *)&(v)->counter) +int arch_atomic_fetch_or(int, atomic_t *); +#define arch_atomic_fetch_or arch_atomic_fetch_or -#define atomic_add(i, v) ((void)__atomic_add_return( (int)(i), (v))) -#define atomic_sub(i, v) ((void)__atomic_add_return(-(int)(i), (v))) -#define atomic_inc(v) ((void)__atomic_add_return( 1, (v))) -#define atomic_dec(v) ((void)__atomic_add_return( -1, (v))) +int arch_atomic_fetch_xor(int, atomic_t *); +#define arch_atomic_fetch_xor arch_atomic_fetch_xor -#define atomic_add_return(i, v) (__atomic_add_return( (int)(i), (v))) -#define atomic_sub_return(i, v) (__atomic_add_return(-(int)(i), (v))) -#define atomic_inc_return(v) (__atomic_add_return( 1, (v))) -#define atomic_dec_return(v) (__atomic_add_return( -1, (v))) +int arch_atomic_cmpxchg(atomic_t *, int, int); +#define arch_atomic_cmpxchg arch_atomic_cmpxchg -#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) +int arch_atomic_xchg(atomic_t *, int); +#define arch_atomic_xchg arch_atomic_xchg -/* - * atomic_inc_and_test - increment and test - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) +int arch_atomic_fetch_add_unless(atomic_t *, int, int); +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless + +void arch_atomic_set(atomic_t *, int); + +#define arch_atomic_set_release(v, i) arch_atomic_set((v), (i)) + +#define arch_atomic_read(v) READ_ONCE((v)->counter) + +#define arch_atomic_add(i, v) ((void)arch_atomic_add_return( (int)(i), (v))) +#define arch_atomic_sub(i, v) ((void)arch_atomic_add_return(-(int)(i), (v))) -#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) -#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) +#define arch_atomic_and(i, v) ((void)arch_atomic_fetch_and((i), (v))) +#define arch_atomic_or(i, v) ((void)arch_atomic_fetch_or((i), (v))) +#define arch_atomic_xor(i, v) ((void)arch_atomic_fetch_xor((i), (v))) -/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() +#define arch_atomic_sub_return(i, v) (arch_atomic_add_return(-(int)(i), (v))) +#define arch_atomic_fetch_sub(i, v) (arch_atomic_fetch_add (-(int)(i), (v))) #endif /* !(__ARCH_SPARC_ATOMIC__) */ diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h index be56a244c9cf..a5e9c37605a7 100644 --- a/arch/sparc/include/asm/atomic_64.h +++ b/arch/sparc/include/asm/atomic_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* atomic.h: Thankfully the V9 is at least reasonable for this * stuff. * @@ -9,109 +10,64 @@ #include <linux/types.h> #include <asm/cmpxchg.h> +#include <asm/barrier.h> -#define ATOMIC_INIT(i) { (i) } #define ATOMIC64_INIT(i) { (i) } -#define atomic_read(v) (*(volatile int *)&(v)->counter) -#define atomic64_read(v) (*(volatile long *)&(v)->counter) +#define arch_atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic64_read(v) READ_ONCE((v)->counter) -#define atomic_set(v, i) (((v)->counter) = i) -#define atomic64_set(v, i) (((v)->counter) = i) +#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) +#define arch_atomic64_set(v, i) WRITE_ONCE(((v)->counter), (i)) -extern void atomic_add(int, atomic_t *); -extern void atomic64_add(long, atomic64_t *); -extern void atomic_sub(int, atomic_t *); -extern void atomic64_sub(long, atomic64_t *); +#define ATOMIC_OP(op) \ +void arch_atomic_##op(int, atomic_t *); \ +void arch_atomic64_##op(s64, atomic64_t *); -extern int atomic_add_ret(int, atomic_t *); -extern long atomic64_add_ret(long, atomic64_t *); -extern int atomic_sub_ret(int, atomic_t *); -extern long atomic64_sub_ret(long, atomic64_t *); +#define ATOMIC_OP_RETURN(op) \ +int arch_atomic_##op##_return(int, atomic_t *); \ +s64 arch_atomic64_##op##_return(s64, atomic64_t *); -#define atomic_dec_return(v) atomic_sub_ret(1, v) -#define atomic64_dec_return(v) atomic64_sub_ret(1, v) +#define ATOMIC_FETCH_OP(op) \ +int arch_atomic_fetch_##op(int, atomic_t *); \ +s64 arch_atomic64_fetch_##op(s64, atomic64_t *); -#define atomic_inc_return(v) atomic_add_ret(1, v) -#define atomic64_inc_return(v) atomic64_add_ret(1, v) +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) -#define atomic_sub_return(i, v) atomic_sub_ret(i, v) -#define atomic64_sub_return(i, v) atomic64_sub_ret(i, v) +ATOMIC_OPS(add) +ATOMIC_OPS(sub) -#define atomic_add_return(i, v) atomic_add_ret(i, v) -#define atomic64_add_return(i, v) atomic64_add_ret(i, v) +#define arch_atomic_add_return arch_atomic_add_return +#define arch_atomic_sub_return arch_atomic_sub_return +#define arch_atomic_fetch_add arch_atomic_fetch_add +#define arch_atomic_fetch_sub arch_atomic_fetch_sub -/* - * atomic_inc_and_test - increment and test - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) -#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) - -#define atomic_sub_and_test(i, v) (atomic_sub_ret(i, v) == 0) -#define atomic64_sub_and_test(i, v) (atomic64_sub_ret(i, v) == 0) - -#define atomic_dec_and_test(v) (atomic_sub_ret(1, v) == 0) -#define atomic64_dec_and_test(v) (atomic64_sub_ret(1, v) == 0) - -#define atomic_inc(v) atomic_add(1, v) -#define atomic64_inc(v) atomic64_add(1, v) - -#define atomic_dec(v) atomic_sub(1, v) -#define atomic64_dec(v) atomic64_sub(1, v) - -#define atomic_add_negative(i, v) (atomic_add_ret(i, v) < 0) -#define atomic64_add_negative(i, v) (atomic64_add_ret(i, v) < 0) - -#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) - -static inline int __atomic_add_unless(atomic_t *v, int a, int u) -{ - int c, old; - c = atomic_read(v); - for (;;) { - if (unlikely(c == (u))) - break; - old = atomic_cmpxchg((v), c, c + (a)); - if (likely(old == c)) - break; - c = old; - } - return c; -} - -#define atomic64_cmpxchg(v, o, n) \ - ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) -#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) - -static inline long atomic64_add_unless(atomic64_t *v, long a, long u) -{ - long c, old; - c = atomic64_read(v); - for (;;) { - if (unlikely(c == (u))) - break; - old = atomic64_cmpxchg((v), c, c + (a)); - if (likely(old == c)) - break; - c = old; - } - return c != (u); -} - -#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) - -extern long atomic64_dec_if_positive(atomic64_t *v); - -/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() +#define arch_atomic64_add_return arch_atomic64_add_return +#define arch_atomic64_sub_return arch_atomic64_sub_return +#define arch_atomic64_fetch_add arch_atomic64_fetch_add +#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub + +#undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) + +#define arch_atomic_fetch_and arch_atomic_fetch_and +#define arch_atomic_fetch_or arch_atomic_fetch_or +#define arch_atomic_fetch_xor arch_atomic_fetch_xor + +#define arch_atomic64_fetch_and arch_atomic64_fetch_and +#define arch_atomic64_fetch_or arch_atomic64_fetch_or +#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor + +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP +#undef ATOMIC_OP_RETURN +#undef ATOMIC_OP + +s64 arch_atomic64_dec_if_positive(atomic64_t *v); +#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive #endif /* !(__ARCH_SPARC64_ATOMIC__) */ diff --git a/arch/sparc/include/asm/auxio.h b/arch/sparc/include/asm/auxio.h index 13dc67f03011..d0a933ed0d04 100644 --- a/arch/sparc/include/asm/auxio.h +++ b/arch/sparc/include/asm/auxio.h @@ -1,5 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_AUXIO_H #define ___ASM_SPARC_AUXIO_H + +#ifndef __ASSEMBLER__ + +extern void __iomem *auxio_register; + +#endif /* ifndef __ASSEMBLER__ */ + #if defined(__sparc__) && defined(__arch64__) #include <asm/auxio_64.h> #else diff --git a/arch/sparc/include/asm/auxio_32.h b/arch/sparc/include/asm/auxio_32.h index 3a319775ae37..db58fa28de9e 100644 --- a/arch/sparc/include/asm/auxio_32.h +++ b/arch/sparc/include/asm/auxio_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * auxio.h: Definitions and code for the Auxiliary I/O register. * @@ -28,14 +29,14 @@ #define AUXIO_FLPY_EJCT 0x02 /* Eject floppy disk. Write only. */ #define AUXIO_LED 0x01 /* On if set, off if unset. Read/Write */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * NOTE: these routines are implementation dependent-- * understand the hardware you are querying! */ -extern void set_auxio(unsigned char bits_on, unsigned char bits_off); -extern unsigned char get_auxio(void); /* .../asm/floppy.h */ +void set_auxio(unsigned char bits_on, unsigned char bits_off); +unsigned char get_auxio(void); /* .../asm/floppy.h */ /* * The following routines are provided for driver-compatibility @@ -74,11 +75,11 @@ do { \ } \ } while (0) -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ /* AUXIO2 (Power Off Control) */ -extern __volatile__ unsigned char * auxio_power_register; +extern volatile u8 __iomem *auxio_power_register; #define AUXIO_POWER_DETECT_FAILURE 32 #define AUXIO_POWER_CLEAR_FAILURE 2 diff --git a/arch/sparc/include/asm/auxio_64.h b/arch/sparc/include/asm/auxio_64.h index f61cd1e3e395..8a4ae07daf16 100644 --- a/arch/sparc/include/asm/auxio_64.h +++ b/arch/sparc/include/asm/auxio_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * auxio.h: Definitions and code for the Auxiliary I/O registers. * @@ -73,9 +74,7 @@ #define AUXIO_PCIO_CPWR_OFF 0x02 /* Courtesy Power Off */ #define AUXIO_PCIO_SPWR_OFF 0x01 /* System Power Off */ -#ifndef __ASSEMBLY__ - -extern void __iomem *auxio_register; +#ifndef __ASSEMBLER__ #define AUXIO_LTE_ON 1 #define AUXIO_LTE_OFF 0 @@ -84,7 +83,7 @@ extern void __iomem *auxio_register; * * on - AUXIO_LTE_ON or AUXIO_LTE_OFF */ -extern void auxio_set_lte(int on); +void auxio_set_lte(int on); #define AUXIO_LED_ON 1 #define AUXIO_LED_OFF 0 @@ -93,8 +92,8 @@ extern void auxio_set_lte(int on); * * on - AUXIO_LED_ON or AUXIO_LED_OFF */ -extern void auxio_set_led(int on); +void auxio_set_led(int on); -#endif /* ifndef __ASSEMBLY__ */ +#endif /* ifndef __ASSEMBLER__ */ #endif /* !(_SPARC64_AUXIO_H) */ diff --git a/arch/sparc/include/asm/backoff.h b/arch/sparc/include/asm/backoff.h index 4e02086b839c..597a22953bc5 100644 --- a/arch/sparc/include/asm/backoff.h +++ b/arch/sparc/include/asm/backoff.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_BACKOFF_H #define _SPARC64_BACKOFF_H @@ -17,7 +18,7 @@ * * When we spin, we try to use an operation that will cause the * current cpu strand to block, and therefore make the core fully - * available to any other other runnable strands. There are two + * available to any other runnable strands. There are two * options, based upon cpu capabilities. * * On all cpus prior to SPARC-T4 we do three dummy reads of the diff --git a/arch/sparc/include/asm/barrier.h b/arch/sparc/include/asm/barrier.h index b25f02a029e0..c10b4164be59 100644 --- a/arch/sparc/include/asm/barrier.h +++ b/arch/sparc/include/asm/barrier.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_BARRIER_H #define ___ASM_SPARC_BARRIER_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/barrier_32.h b/arch/sparc/include/asm/barrier_32.h index c1b76654ee76..304f9c3f7c4a 100644 --- a/arch/sparc/include/asm/barrier_32.h +++ b/arch/sparc/include/asm/barrier_32.h @@ -1,15 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_BARRIER_H #define __SPARC_BARRIER_H -/* XXX Change this if we ever use a PSO mode kernel. */ -#define mb() __asm__ __volatile__ ("" : : : "memory") -#define rmb() mb() -#define wmb() mb() -#define read_barrier_depends() do { } while(0) -#define set_mb(__var, __value) do { __var = __value; mb(); } while(0) -#define smp_mb() __asm__ __volatile__("":::"memory") -#define smp_rmb() __asm__ __volatile__("":::"memory") -#define smp_wmb() __asm__ __volatile__("":::"memory") -#define smp_read_barrier_depends() do { } while(0) +#include <asm-generic/barrier.h> #endif /* !(__SPARC_BARRIER_H) */ diff --git a/arch/sparc/include/asm/barrier_64.h b/arch/sparc/include/asm/barrier_64.h index 95d45986f908..9fb148bd3c97 100644 --- a/arch/sparc/include/asm/barrier_64.h +++ b/arch/sparc/include/asm/barrier_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC64_BARRIER_H #define __SPARC64_BARRIER_H @@ -37,20 +38,24 @@ do { __asm__ __volatile__("ba,pt %%xcc, 1f\n\t" \ #define rmb() __asm__ __volatile__("":::"memory") #define wmb() __asm__ __volatile__("":::"memory") -#define read_barrier_depends() do { } while(0) -#define set_mb(__var, __value) \ - do { __var = __value; membar_safe("#StoreLoad"); } while(0) +#define __smp_store_release(p, v) \ +do { \ + compiletime_assert_atomic_type(*p); \ + barrier(); \ + WRITE_ONCE(*p, v); \ +} while (0) + +#define __smp_load_acquire(p) \ +({ \ + typeof(*p) ___p1 = READ_ONCE(*p); \ + compiletime_assert_atomic_type(*p); \ + barrier(); \ + ___p1; \ +}) -#ifdef CONFIG_SMP -#define smp_mb() mb() -#define smp_rmb() rmb() -#define smp_wmb() wmb() -#else -#define smp_mb() __asm__ __volatile__("":::"memory") -#define smp_rmb() __asm__ __volatile__("":::"memory") -#define smp_wmb() __asm__ __volatile__("":::"memory") -#endif +#define __smp_mb__before_atomic() barrier() +#define __smp_mb__after_atomic() barrier() -#define smp_read_barrier_depends() do { } while(0) +#include <asm-generic/barrier.h> #endif /* !(__SPARC64_BARRIER_H) */ diff --git a/arch/sparc/include/asm/bbc.h b/arch/sparc/include/asm/bbc.h index 423a85800aae..00a4f116144d 100644 --- a/arch/sparc/include/asm/bbc.h +++ b/arch/sparc/include/asm/bbc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * bbc.h: Defines for BootBus Controller found on UltraSPARC-III * systems. diff --git a/arch/sparc/include/asm/bitext.h b/arch/sparc/include/asm/bitext.h index 297b2f2fcb49..2c2a2d85e8ec 100644 --- a/arch/sparc/include/asm/bitext.h +++ b/arch/sparc/include/asm/bitext.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * bitext.h: Bit string operations on the sparc, specific to architecture. * @@ -20,8 +21,8 @@ struct bit_map { int num_colors; }; -extern int bit_map_string_get(struct bit_map *t, int len, int align); -extern void bit_map_clear(struct bit_map *t, int offset, int len); -extern void bit_map_init(struct bit_map *t, unsigned long *map, int size); +int bit_map_string_get(struct bit_map *t, int len, int align); +void bit_map_clear(struct bit_map *t, int offset, int len); +void bit_map_init(struct bit_map *t, unsigned long *map, int size); #endif /* defined(_SPARC_BITEXT_H) */ diff --git a/arch/sparc/include/asm/bitops.h b/arch/sparc/include/asm/bitops.h index b1edd94bd64f..4c431d274926 100644 --- a/arch/sparc/include/asm/bitops.h +++ b/arch/sparc/include/asm/bitops.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_BITOPS_H #define ___ASM_SPARC_BITOPS_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/bitops_32.h b/arch/sparc/include/asm/bitops_32.h index 25a676653d45..3448c191b484 100644 --- a/arch/sparc/include/asm/bitops_32.h +++ b/arch/sparc/include/asm/bitops_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * bitops.h: Bit string operations on the Sparc. * @@ -18,9 +19,9 @@ #error only <linux/bitops.h> can be included directly #endif -extern unsigned long ___set_bit(unsigned long *addr, unsigned long mask); -extern unsigned long ___clear_bit(unsigned long *addr, unsigned long mask); -extern unsigned long ___change_bit(unsigned long *addr, unsigned long mask); +unsigned long sp32___set_bit(unsigned long *addr, unsigned long mask); +unsigned long sp32___clear_bit(unsigned long *addr, unsigned long mask); +unsigned long sp32___change_bit(unsigned long *addr, unsigned long mask); /* * Set bit 'nr' in 32-bit quantity at address 'addr' where bit '0' @@ -35,7 +36,7 @@ static inline int test_and_set_bit(unsigned long nr, volatile unsigned long *add ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - return ___set_bit(ADDR, mask) != 0; + return sp32___set_bit(ADDR, mask) != 0; } static inline void set_bit(unsigned long nr, volatile unsigned long *addr) @@ -45,7 +46,7 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *addr) ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - (void) ___set_bit(ADDR, mask); + (void) sp32___set_bit(ADDR, mask); } static inline int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) @@ -55,7 +56,7 @@ static inline int test_and_clear_bit(unsigned long nr, volatile unsigned long *a ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - return ___clear_bit(ADDR, mask) != 0; + return sp32___clear_bit(ADDR, mask) != 0; } static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) @@ -65,7 +66,7 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - (void) ___clear_bit(ADDR, mask); + (void) sp32___clear_bit(ADDR, mask); } static inline int test_and_change_bit(unsigned long nr, volatile unsigned long *addr) @@ -75,7 +76,7 @@ static inline int test_and_change_bit(unsigned long nr, volatile unsigned long * ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - return ___change_bit(ADDR, mask) != 0; + return sp32___change_bit(ADDR, mask) != 0; } static inline void change_bit(unsigned long nr, volatile unsigned long *addr) @@ -85,14 +86,11 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr) ADDR = ((unsigned long *) addr) + (nr >> 5); mask = 1 << (nr & 31); - (void) ___change_bit(ADDR, mask); + (void) sp32___change_bit(ADDR, mask); } #include <asm-generic/bitops/non-atomic.h> -#define smp_mb__before_clear_bit() do { } while(0) -#define smp_mb__after_clear_bit() do { } while(0) - #include <asm-generic/bitops/ffz.h> #include <asm-generic/bitops/__ffs.h> #include <asm-generic/bitops/sched.h> @@ -102,7 +100,6 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr) #include <asm-generic/bitops/fls64.h> #include <asm-generic/bitops/hweight.h> #include <asm-generic/bitops/lock.h> -#include <asm-generic/bitops/find.h> #include <asm-generic/bitops/le.h> #include <asm-generic/bitops/ext2-atomic.h> diff --git a/arch/sparc/include/asm/bitops_64.h b/arch/sparc/include/asm/bitops_64.h index 29011cc0e4be..2c7d33b3ec2e 100644 --- a/arch/sparc/include/asm/bitops_64.h +++ b/arch/sparc/include/asm/bitops_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * bitops.h: Bit string operations on the V9. * @@ -13,27 +14,26 @@ #include <linux/compiler.h> #include <asm/byteorder.h> +#include <asm/barrier.h> -extern int test_and_set_bit(unsigned long nr, volatile unsigned long *addr); -extern int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr); -extern int test_and_change_bit(unsigned long nr, volatile unsigned long *addr); -extern void set_bit(unsigned long nr, volatile unsigned long *addr); -extern void clear_bit(unsigned long nr, volatile unsigned long *addr); -extern void change_bit(unsigned long nr, volatile unsigned long *addr); +int test_and_set_bit(unsigned long nr, volatile unsigned long *addr); +int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr); +int test_and_change_bit(unsigned long nr, volatile unsigned long *addr); +void set_bit(unsigned long nr, volatile unsigned long *addr); +void clear_bit(unsigned long nr, volatile unsigned long *addr); +void change_bit(unsigned long nr, volatile unsigned long *addr); -#include <asm-generic/bitops/non-atomic.h> +int __attribute_const__ fls(unsigned int word); +int __attribute_const__ __fls(unsigned long word); -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() +#include <asm-generic/bitops/non-atomic.h> -#include <asm-generic/bitops/fls.h> -#include <asm-generic/bitops/__fls.h> #include <asm-generic/bitops/fls64.h> #ifdef __KERNEL__ -extern int ffs(int x); -extern unsigned long __ffs(unsigned long); +int __attribute_const__ ffs(int x); +unsigned long __attribute_const__ __ffs(unsigned long); #include <asm-generic/bitops/ffz.h> #include <asm-generic/bitops/sched.h> @@ -43,17 +43,15 @@ extern unsigned long __ffs(unsigned long); * of bits set) of a N-bit word */ -extern unsigned long __arch_hweight64(__u64 w); -extern unsigned int __arch_hweight32(unsigned int w); -extern unsigned int __arch_hweight16(unsigned int w); -extern unsigned int __arch_hweight8(unsigned int w); +unsigned long __arch_hweight64(__u64 w); +unsigned int __arch_hweight32(unsigned int w); +unsigned int __arch_hweight16(unsigned int w); +unsigned int __arch_hweight8(unsigned int w); #include <asm-generic/bitops/const_hweight.h> #include <asm-generic/bitops/lock.h> #endif /* __KERNEL__ */ -#include <asm-generic/bitops/find.h> - #ifdef __KERNEL__ #include <asm-generic/bitops/le.h> diff --git a/arch/sparc/include/asm/btext.h b/arch/sparc/include/asm/btext.h index 9b2bc6b6ed0a..01cdc4265733 100644 --- a/arch/sparc/include/asm/btext.h +++ b/arch/sparc/include/asm/btext.h @@ -1,6 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_BTEXT_H #define _SPARC_BTEXT_H -extern int btext_find_display(void); +int btext_find_display(void); #endif /* _SPARC_BTEXT_H */ diff --git a/arch/sparc/include/asm/bug.h b/arch/sparc/include/asm/bug.h index 6bd9f43cb5a5..ea53e418f6c0 100644 --- a/arch/sparc/include/asm/bug.h +++ b/arch/sparc/include/asm/bug.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_BUG_H #define _SPARC_BUG_H @@ -5,13 +6,17 @@ #include <linux/compiler.h> #ifdef CONFIG_DEBUG_BUGVERBOSE -extern void do_BUG(const char *file, int line); +void do_BUG(const char *file, int line); #define BUG() do { \ do_BUG(__FILE__, __LINE__); \ + barrier_before_unreachable(); \ __builtin_trap(); \ } while (0) #else -#define BUG() __builtin_trap() +#define BUG() do { \ + barrier_before_unreachable(); \ + __builtin_trap(); \ +} while (0) #endif #define HAVE_ARCH_BUG @@ -20,6 +25,6 @@ extern void do_BUG(const char *file, int line); #include <asm-generic/bug.h> struct pt_regs; -extern void die_if_kernel(char *str, struct pt_regs *regs) __attribute__ ((noreturn)); +void __noreturn die_if_kernel(char *str, struct pt_regs *regs); #endif diff --git a/arch/sparc/include/asm/bugs.h b/arch/sparc/include/asm/bugs.h deleted file mode 100644 index 61d86bbbe2b2..000000000000 --- a/arch/sparc/include/asm/bugs.h +++ /dev/null @@ -1,17 +0,0 @@ -/* include/asm/bugs.h: Sparc probes for various bugs. - * - * Copyright (C) 1996, 2007 David S. Miller (davem@davemloft.net) - */ - -#ifdef CONFIG_SPARC32 -#include <asm/cpudata.h> -#endif - -extern unsigned long loops_per_jiffy; - -static void __init check_bugs(void) -{ -#if defined(CONFIG_SPARC32) && !defined(CONFIG_SMP) - cpu_data(0).udelay_val = loops_per_jiffy; -#endif -} diff --git a/arch/sparc/include/asm/cache.h b/arch/sparc/include/asm/cache.h index 5bb6991b4857..e62fd0e72606 100644 --- a/arch/sparc/include/asm/cache.h +++ b/arch/sparc/include/asm/cache.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* cache.h: Cache specific code for the Sparc. These include flushing * and direct tag/data line access. * @@ -20,6 +21,6 @@ #define SMP_CACHE_BYTES (1 << SMP_CACHE_BYTES_SHIFT) -#define __read_mostly __attribute__((__section__(".data..read_mostly"))) +#define __read_mostly __section(".data..read_mostly") #endif /* !(_SPARC_CACHE_H) */ diff --git a/arch/sparc/include/asm/cacheflush.h b/arch/sparc/include/asm/cacheflush.h index f6c4839b8388..881ac76eab93 100644 --- a/arch/sparc/include/asm/cacheflush.h +++ b/arch/sparc/include/asm/cacheflush.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_CACHEFLUSH_H #define ___ASM_SPARC_CACHEFLUSH_H diff --git a/arch/sparc/include/asm/cacheflush_32.h b/arch/sparc/include/asm/cacheflush_32.h index bb014c24f318..9fee0ccfccb8 100644 --- a/arch/sparc/include/asm/cacheflush_32.h +++ b/arch/sparc/include/asm/cacheflush_32.h @@ -1,6 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_CACHEFLUSH_H #define _SPARC_CACHEFLUSH_H +#include <linux/page-flags.h> #include <asm/cachetlb_32.h> #define flush_cache_all() \ @@ -14,9 +16,6 @@ #define flush_cache_page(vma,addr,pfn) \ sparc32_cachetlb_ops->cache_page(vma, addr) #define flush_icache_range(start, end) do { } while (0) -#define flush_icache_page(vma, pg) do { } while (0) - -#define flush_icache_user_range(vma,pg,adr,len) do { } while (0) #define copy_to_user_page(vma, page, vaddr, dst, src, len) \ do { \ @@ -36,14 +35,20 @@ #define flush_page_for_dma(addr) \ sparc32_cachetlb_ops->page_for_dma(addr) -extern void sparc_flush_page_to_ram(struct page *page); +void sparc_flush_page_to_ram(struct page *page); +void sparc_flush_folio_to_ram(struct folio *folio); #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 -#define flush_dcache_page(page) sparc_flush_page_to_ram(page) +#define flush_dcache_folio(folio) sparc_flush_folio_to_ram(folio) +static inline void flush_dcache_page(struct page *page) +{ + flush_dcache_folio(page_folio(page)); +} #define flush_dcache_mmap_lock(mapping) do { } while (0) #define flush_dcache_mmap_unlock(mapping) do { } while (0) #define flush_cache_vmap(start, end) flush_cache_all() +#define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) flush_cache_all() /* When a context switch happens we must flush all user windows so that @@ -51,8 +56,8 @@ extern void sparc_flush_page_to_ram(struct page *page); * way the windows are all clean for the next process and the stack * frames are up to date. */ -extern void flush_user_windows(void); -extern void kill_user_windows(void); -extern void flushw_all(void); +void flush_user_windows(void); +void kill_user_windows(void); +void flushw_all(void); #endif /* _SPARC_CACHEFLUSH_H */ diff --git a/arch/sparc/include/asm/cacheflush_64.h b/arch/sparc/include/asm/cacheflush_64.h index 301736d9e7a1..06092572c045 100644 --- a/arch/sparc/include/asm/cacheflush_64.h +++ b/arch/sparc/include/asm/cacheflush_64.h @@ -1,16 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_CACHEFLUSH_H #define _SPARC64_CACHEFLUSH_H #include <asm/page.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/mm.h> /* Cache flush operations. */ #define flushw_all() __asm__ __volatile__("flushw") -extern void __flushw_user(void); +void __flushw_user(void); #define flushw_user() __flushw_user() #define flush_user_windows flushw_user @@ -30,29 +31,31 @@ extern void __flushw_user(void); * use block commit stores (which invalidate icache lines) during * module load, so we need this. */ -extern void flush_icache_range(unsigned long start, unsigned long end); -extern void __flush_icache_page(unsigned long); +void flush_icache_range(unsigned long start, unsigned long end); +void __flush_icache_page(unsigned long); -extern void __flush_dcache_page(void *addr, int flush_icache); -extern void flush_dcache_page_impl(struct page *page); +void __flush_dcache_page(void *addr, int flush_icache); +void flush_dcache_folio_impl(struct folio *folio); #ifdef CONFIG_SMP -extern void smp_flush_dcache_page_impl(struct page *page, int cpu); -extern void flush_dcache_page_all(struct mm_struct *mm, struct page *page); +void smp_flush_dcache_folio_impl(struct folio *folio, int cpu); +void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio); #else -#define smp_flush_dcache_page_impl(page,cpu) flush_dcache_page_impl(page) -#define flush_dcache_page_all(mm,page) flush_dcache_page_impl(page) +#define smp_flush_dcache_folio_impl(folio, cpu) flush_dcache_folio_impl(folio) +#define flush_dcache_folio_all(mm, folio) flush_dcache_folio_impl(folio) #endif -extern void __flush_dcache_range(unsigned long start, unsigned long end); +void __flush_dcache_range(unsigned long start, unsigned long end); #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 -extern void flush_dcache_page(struct page *page); +void flush_dcache_folio(struct folio *folio); +#define flush_dcache_folio flush_dcache_folio +static inline void flush_dcache_page(struct page *page) +{ + flush_dcache_folio(page_folio(page)); +} -#define flush_icache_page(vma, pg) do { } while(0) -#define flush_icache_user_range(vma,pg,adr,len) do { } while (0) - -extern void flush_ptrace_access(struct vm_area_struct *, struct page *, - unsigned long uaddr, void *kaddr, - unsigned long len, int write); +void flush_ptrace_access(struct vm_area_struct *, struct page *, + unsigned long uaddr, void *kaddr, + unsigned long len, int write); #define copy_to_user_page(vma, page, vaddr, dst, src, len) \ do { \ @@ -72,13 +75,9 @@ extern void flush_ptrace_access(struct vm_area_struct *, struct page *, #define flush_dcache_mmap_unlock(mapping) do { } while (0) #define flush_cache_vmap(start, end) do { } while (0) +#define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) do { } while (0) -#ifdef CONFIG_DEBUG_PAGEALLOC -/* internal debugging function */ -void kernel_map_pages(struct page *page, int numpages, int enable); -#endif - -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* _SPARC64_CACHEFLUSH_H */ diff --git a/arch/sparc/include/asm/cachetlb_32.h b/arch/sparc/include/asm/cachetlb_32.h index efb19889a083..534da70c6357 100644 --- a/arch/sparc/include/asm/cachetlb_32.h +++ b/arch/sparc/include/asm/cachetlb_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_CACHETLB_H #define _SPARC_CACHETLB_H diff --git a/arch/sparc/include/asm/cachetype.h b/arch/sparc/include/asm/cachetype.h new file mode 100644 index 000000000000..caf1c0045892 --- /dev/null +++ b/arch/sparc/include/asm/cachetype.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_SPARC_CACHETYPE_H +#define __ASM_SPARC_CACHETYPE_H + +#include <asm/page.h> + +#ifdef CONFIG_SPARC32 +extern int vac_cache_size; +#define cpu_dcache_is_aliasing() (vac_cache_size > PAGE_SIZE) +#else +#define cpu_dcache_is_aliasing() (L1DCACHE_SIZE > PAGE_SIZE) +#endif + +#endif diff --git a/arch/sparc/include/asm/chafsr.h b/arch/sparc/include/asm/chafsr.h index 85c69b38220b..01540eca1671 100644 --- a/arch/sparc/include/asm/chafsr.h +++ b/arch/sparc/include/asm/chafsr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_CHAFSR_H #define _SPARC64_CHAFSR_H diff --git a/arch/sparc/include/asm/checksum.h b/arch/sparc/include/asm/checksum.h index 7ac0d7497bc5..f2ac13323b6d 100644 --- a/arch/sparc/include/asm/checksum.h +++ b/arch/sparc/include/asm/checksum.h @@ -1,5 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_CHECKSUM_H #define ___ASM_SPARC_CHECKSUM_H +#define _HAVE_ARCH_CSUM_AND_COPY +#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER +#define HAVE_CSUM_COPY_USER #if defined(__sparc__) && defined(__arch64__) #include <asm/checksum_64.h> #else diff --git a/arch/sparc/include/asm/checksum_32.h b/arch/sparc/include/asm/checksum_32.h index bdbda1453aa9..ce11e0ad80c7 100644 --- a/arch/sparc/include/asm/checksum_32.h +++ b/arch/sparc/include/asm/checksum_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_CHECKSUM_H #define __SPARC_CHECKSUM_H @@ -16,7 +17,7 @@ */ #include <linux/in6.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> /* computes the checksum of a memory block at buff, length len, * and adds in "sum" (32-bit) @@ -29,7 +30,7 @@ * * it's best to have buff aligned on a 32-bit boundary */ -extern __wsum csum_partial(const void *buff, int len, __wsum sum); +__wsum csum_partial(const void *buff, int len, __wsum sum); /* the same as csum_partial, but copies from fs:src while it * checksums @@ -38,10 +39,10 @@ extern __wsum csum_partial(const void *buff, int len, __wsum sum); * better 64-bit) boundary */ -extern unsigned int __csum_partial_copy_sparc_generic (const unsigned char *, unsigned char *); +unsigned int __csum_partial_copy_sparc_generic (const unsigned char *, unsigned char *); static inline __wsum -csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum) +csum_partial_copy_nocheck(const void *src, void *dst, int len) { register unsigned int ret asm("o0") = (unsigned int)src; register char *d asm("o1") = dst; @@ -49,9 +50,9 @@ csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum) __asm__ __volatile__ ( "call __csum_partial_copy_sparc_generic\n\t" - " mov %6, %%g7\n" + " mov -1, %%g7\n" : "=&r" (ret), "=&r" (d), "=&r" (l) - : "0" (ret), "1" (d), "2" (l), "r" (sum) + : "0" (ret), "1" (d), "2" (l) : "o2", "o3", "o4", "o5", "o7", "g2", "g3", "g4", "g5", "g7", "memory", "cc"); @@ -59,62 +60,21 @@ csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum) } static inline __wsum -csum_partial_copy_from_user(const void __user *src, void *dst, int len, - __wsum sum, int *err) - { - register unsigned long ret asm("o0") = (unsigned long)src; - register char *d asm("o1") = dst; - register int l asm("g1") = len; - register __wsum s asm("g7") = sum; - - __asm__ __volatile__ ( - ".section __ex_table,#alloc\n\t" - ".align 4\n\t" - ".word 1f,2\n\t" - ".previous\n" - "1:\n\t" - "call __csum_partial_copy_sparc_generic\n\t" - " st %8, [%%sp + 64]\n" - : "=&r" (ret), "=&r" (d), "=&r" (l), "=&r" (s) - : "0" (ret), "1" (d), "2" (l), "3" (s), "r" (err) - : "o2", "o3", "o4", "o5", "o7", "g2", "g3", "g4", "g5", - "cc", "memory"); - return (__force __wsum)ret; +csum_and_copy_from_user(const void __user *src, void *dst, int len) +{ + if (unlikely(!access_ok(src, len))) + return 0; + return csum_partial_copy_nocheck((__force void *)src, dst, len); } static inline __wsum -csum_partial_copy_to_user(const void *src, void __user *dst, int len, - __wsum sum, int *err) +csum_and_copy_to_user(const void *src, void __user *dst, int len) { - if (!access_ok (VERIFY_WRITE, dst, len)) { - *err = -EFAULT; - return sum; - } else { - register unsigned long ret asm("o0") = (unsigned long)src; - register char __user *d asm("o1") = dst; - register int l asm("g1") = len; - register __wsum s asm("g7") = sum; - - __asm__ __volatile__ ( - ".section __ex_table,#alloc\n\t" - ".align 4\n\t" - ".word 1f,1\n\t" - ".previous\n" - "1:\n\t" - "call __csum_partial_copy_sparc_generic\n\t" - " st %8, [%%sp + 64]\n" - : "=&r" (ret), "=&r" (d), "=&r" (l), "=&r" (s) - : "0" (ret), "1" (d), "2" (l), "3" (s), "r" (err) - : "o2", "o3", "o4", "o5", "o7", - "g2", "g3", "g4", "g5", - "cc", "memory"); - return (__force __wsum)ret; - } + if (!access_ok(dst, len)) + return 0; + return csum_partial_copy_nocheck(src, (__force void *)dst, len); } -#define HAVE_CSUM_COPY_USER -#define csum_and_copy_to_user csum_partial_copy_to_user - /* ihl is always 5 or greater, almost always is 5, and iph is word aligned * the majority of the time. */ @@ -170,9 +130,8 @@ static inline __sum16 csum_fold(__wsum sum) } static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, - unsigned short len, - unsigned short proto, - __wsum sum) + __u32 len, __u8 proto, + __wsum sum) { __asm__ __volatile__("addcc\t%1, %0, %0\n\t" "addxcc\t%2, %0, %0\n\t" @@ -190,9 +149,8 @@ static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, * returns a 16-bit checksum, already complemented */ static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, - unsigned short len, - unsigned short proto, - __wsum sum) + __u32 len, __u8 proto, + __wsum sum) { return csum_fold(csum_tcpudp_nofold(saddr,daddr,len,proto,sum)); } @@ -201,8 +159,7 @@ static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, static inline __sum16 csum_ipv6_magic(const struct in6_addr *saddr, const struct in6_addr *daddr, - __u32 len, unsigned short proto, - __wsum sum) + __u32 len, __u8 proto, __wsum sum) { __asm__ __volatile__ ( "addcc %3, %4, %%g4\n\t" @@ -238,4 +195,16 @@ static inline __sum16 ip_compute_csum(const void *buff, int len) return csum_fold(csum_partial(buff, len, 0)); } +#define HAVE_ARCH_CSUM_ADD +static inline __wsum csum_add(__wsum csum, __wsum addend) +{ + __asm__ __volatile__( + "addcc %0, %1, %0\n" + "addx %0, %%g0, %0" + : "=r" (csum) + : "r" (addend), "0" (csum)); + + return csum; +} + #endif /* !(__SPARC_CHECKSUM_H) */ diff --git a/arch/sparc/include/asm/checksum_64.h b/arch/sparc/include/asm/checksum_64.h index 019b9615e43c..d6b59461e064 100644 --- a/arch/sparc/include/asm/checksum_64.h +++ b/arch/sparc/include/asm/checksum_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC64_CHECKSUM_H #define __SPARC64_CHECKSUM_H @@ -16,7 +17,7 @@ */ #include <linux/in6.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> /* computes the checksum of a memory block at buff, length len, * and adds in "sum" (32-bit) @@ -29,7 +30,7 @@ * * it's best to have buff aligned on a 32-bit boundary */ -extern __wsum csum_partial(const void * buff, int len, __wsum sum); +__wsum csum_partial(const void * buff, int len, __wsum sum); /* the same as csum_partial, but copies from user space while it * checksums @@ -37,47 +38,14 @@ extern __wsum csum_partial(const void * buff, int len, __wsum sum); * here even more important to align src and dst on a 32-bit (or even * better 64-bit) boundary */ -extern __wsum csum_partial_copy_nocheck(const void *src, void *dst, - int len, __wsum sum); - -extern long __csum_partial_copy_from_user(const void __user *src, - void *dst, int len, - __wsum sum); - -static inline __wsum -csum_partial_copy_from_user(const void __user *src, - void *dst, int len, - __wsum sum, int *err) -{ - long ret = __csum_partial_copy_from_user(src, dst, len, sum); - if (ret < 0) - *err = -EFAULT; - return (__force __wsum) ret; -} - -/* - * Copy and checksum to user - */ -#define HAVE_CSUM_COPY_USER -extern long __csum_partial_copy_to_user(const void *src, - void __user *dst, int len, - __wsum sum); - -static inline __wsum -csum_and_copy_to_user(const void *src, - void __user *dst, int len, - __wsum sum, int *err) -{ - long ret = __csum_partial_copy_to_user(src, dst, len, sum); - if (ret < 0) - *err = -EFAULT; - return (__force __wsum) ret; -} +__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len); +__wsum csum_and_copy_from_user(const void __user *src, void *dst, int len); +__wsum csum_and_copy_to_user(const void *src, void __user *dst, int len); /* ihl is always 5 or greater, almost always is 5, and iph is word aligned * the majority of the time. */ -extern __sum16 ip_fast_csum(const void *iph, unsigned int ihl); +__sum16 ip_fast_csum(const void *iph, unsigned int ihl); /* Fold a partial checksum without adding pseudo headers. */ static inline __sum16 csum_fold(__wsum sum) @@ -96,9 +64,8 @@ static inline __sum16 csum_fold(__wsum sum) } static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, - unsigned int len, - unsigned short proto, - __wsum sum) + __u32 len, __u8 proto, + __wsum sum) { __asm__ __volatile__( " addcc %1, %0, %0\n" @@ -116,9 +83,8 @@ static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, * returns a 16-bit checksum, already complemented */ static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, - unsigned short len, - unsigned short proto, - __wsum sum) + __u32 len, __u8 proto, + __wsum sum) { return csum_fold(csum_tcpudp_nofold(saddr,daddr,len,proto,sum)); } @@ -127,8 +93,7 @@ static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, static inline __sum16 csum_ipv6_magic(const struct in6_addr *saddr, const struct in6_addr *daddr, - __u32 len, unsigned short proto, - __wsum sum) + __u32 len, __u8 proto, __wsum sum) { __asm__ __volatile__ ( " addcc %3, %4, %%g7\n" @@ -164,4 +129,16 @@ static inline __sum16 ip_compute_csum(const void *buff, int len) return csum_fold(csum_partial(buff, len, 0)); } +#define HAVE_ARCH_CSUM_ADD +static inline __wsum csum_add(__wsum csum, __wsum addend) +{ + __asm__ __volatile__( + "addcc %0, %1, %0\n" + "addx %0, %%g0, %0" + : "=r" (csum) + : "r" (addend), "0" (csum)); + + return csum; +} + #endif /* !(__SPARC64_CHECKSUM_H) */ diff --git a/arch/sparc/include/asm/chmctrl.h b/arch/sparc/include/asm/chmctrl.h index 859b4a4b0d30..96f043cb2c3e 100644 --- a/arch/sparc/include/asm/chmctrl.h +++ b/arch/sparc/include/asm/chmctrl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_CHMCTRL_H #define _SPARC64_CHMCTRL_H diff --git a/arch/sparc/include/asm/clocksource.h b/arch/sparc/include/asm/clocksource.h new file mode 100644 index 000000000000..d63ef224befe --- /dev/null +++ b/arch/sparc/include/asm/clocksource.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#ifndef _ASM_SPARC_CLOCKSOURCE_H +#define _ASM_SPARC_CLOCKSOURCE_H + +/* VDSO clocksources */ +#define VCLOCK_NONE 0 /* Nothing userspace can do. */ +#define VCLOCK_TICK 1 /* Use %tick. */ +#define VCLOCK_STICK 2 /* Use %stick. */ + +struct arch_clocksource_data { + int vclock_mode; +}; + +#endif /* _ASM_SPARC_CLOCKSOURCE_H */ diff --git a/arch/sparc/include/asm/cmpxchg.h b/arch/sparc/include/asm/cmpxchg.h index 9355893efa52..b7955c6b91fc 100644 --- a/arch/sparc/include/asm/cmpxchg.h +++ b/arch/sparc/include/asm/cmpxchg.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_CMPXCHG_H #define ___ASM_SPARC_CMPXCHG_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/cmpxchg_32.h b/arch/sparc/include/asm/cmpxchg_32.h index 1fae1a02e3c2..8c1a3ca34eeb 100644 --- a/arch/sparc/include/asm/cmpxchg_32.h +++ b/arch/sparc/include/asm/cmpxchg_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* 32-bit atomic xchg() and cmpxchg() definitions. * * Copyright (C) 1996 David S. Miller (davem@davemloft.net) @@ -11,28 +12,20 @@ #ifndef __ARCH_SPARC_CMPXCHG__ #define __ARCH_SPARC_CMPXCHG__ -static inline unsigned long xchg_u32(__volatile__ unsigned long *m, unsigned long val) -{ - __asm__ __volatile__("swap [%2], %0" - : "=&r" (val) - : "0" (val), "r" (m) - : "memory"); - return val; -} +unsigned long __xchg_u32(volatile u32 *m, u32 new); +void __xchg_called_with_bad_pointer(void); -extern void __xchg_called_with_bad_pointer(void); - -static inline unsigned long __xchg(unsigned long x, __volatile__ void * ptr, int size) +static __always_inline unsigned long __arch_xchg(unsigned long x, __volatile__ void * ptr, int size) { switch (size) { case 4: - return xchg_u32(ptr, x); + return __xchg_u32(ptr, x); } __xchg_called_with_bad_pointer(); return x; } -#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) +#define arch_xchg(ptr,x) ({(__typeof__(*(ptr)))__arch_xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));}) /* Emulate cmpxchg() the same way we emulate atomics, * by hashing the object address and indexing into an array @@ -42,28 +35,25 @@ static inline unsigned long __xchg(unsigned long x, __volatile__ void * ptr, int * * Cribbed from <asm-parisc/atomic.h> */ -#define __HAVE_ARCH_CMPXCHG 1 /* bug catcher for when unsupported size is used - won't link */ -extern void __cmpxchg_called_with_bad_pointer(void); -/* we only need to support cmpxchg of a u32 on sparc */ -extern unsigned long __cmpxchg_u32(volatile u32 *m, u32 old, u32 new_); +void __cmpxchg_called_with_bad_pointer(void); +u8 __cmpxchg_u8(volatile u8 *m, u8 old, u8 new_); +u16 __cmpxchg_u16(volatile u16 *m, u16 old, u16 new_); +u32 __cmpxchg_u32(volatile u32 *m, u32 old, u32 new_); /* don't worry...optimizer will get rid of most of this */ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new_, int size) { - switch (size) { - case 4: - return __cmpxchg_u32((u32 *)ptr, (u32)old, (u32)new_); - default: - __cmpxchg_called_with_bad_pointer(); - break; - } - return old; + return + size == 1 ? __cmpxchg_u8(ptr, old, new_) : + size == 2 ? __cmpxchg_u16(ptr, old, new_) : + size == 4 ? __cmpxchg_u32(ptr, old, new_) : + (__cmpxchg_called_with_bad_pointer(), old); } -#define cmpxchg(ptr, o, n) \ +#define arch_cmpxchg(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -71,15 +61,18 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new_, int size) (unsigned long)_n_, sizeof(*(ptr))); \ }) +u64 __cmpxchg_u64(volatile u64 *ptr, u64 old, u64 new); +#define arch_cmpxchg64(ptr, old, new) __cmpxchg_u64(ptr, old, new) + #include <asm-generic/cmpxchg-local.h> /* * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make * them available. */ -#define cmpxchg_local(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\ +#define arch_cmpxchg_local(ptr, o, n) \ + ((__typeof__(*(ptr)))__generic_cmpxchg_local((ptr), (unsigned long)(o),\ (unsigned long)(n), sizeof(*(ptr)))) -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) +#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) #endif /* __ARCH_SPARC_CMPXCHG__ */ diff --git a/arch/sparc/include/asm/cmpxchg_64.h b/arch/sparc/include/asm/cmpxchg_64.h index 4adefe8e2885..3de25262c411 100644 --- a/arch/sparc/include/asm/cmpxchg_64.h +++ b/arch/sparc/include/asm/cmpxchg_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* 64-bit atomic xchg() and cmpxchg() definitions. * * Copyright (C) 1996, 1997, 2000 David S. Miller (davem@redhat.com) @@ -6,6 +7,17 @@ #ifndef __ARCH_SPARC64_CMPXCHG__ #define __ARCH_SPARC64_CMPXCHG__ +static inline unsigned long +__cmpxchg_u32(volatile int *m, int old, int new) +{ + __asm__ __volatile__("cas [%2], %3, %0" + : "=&r" (new) + : "0" (new), "r" (m), "r" (old) + : "memory"); + + return new; +} + static inline unsigned long xchg32(__volatile__ unsigned int *m, unsigned int val) { unsigned long tmp1, tmp2; @@ -40,14 +52,47 @@ static inline unsigned long xchg64(__volatile__ unsigned long *m, unsigned long return val; } -#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) +#define arch_xchg(ptr,x) \ +({ __typeof__(*(ptr)) __ret; \ + __ret = (__typeof__(*(ptr))) \ + __arch_xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \ + __ret; \ +}) + +void __xchg_called_with_bad_pointer(void); -extern void __xchg_called_with_bad_pointer(void); +/* + * Use 4 byte cas instruction to achieve 2 byte xchg. Main logic + * here is to get the bit shift of the byte we are interested in. + * The XOR is handy for reversing the bits for big-endian byte order. + */ +static inline unsigned long +xchg16(__volatile__ unsigned short *m, unsigned short val) +{ + unsigned long maddr = (unsigned long)m; + int bit_shift = (((unsigned long)m & 2) ^ 2) << 3; + unsigned int mask = 0xffff << bit_shift; + unsigned int *ptr = (unsigned int *) (maddr & ~2); + unsigned int old32, new32, load32; + + /* Read the old value */ + load32 = *ptr; + + do { + old32 = load32; + new32 = (load32 & (~mask)) | val << bit_shift; + load32 = __cmpxchg_u32(ptr, old32, new32); + } while (load32 != old32); + + return (load32 & mask) >> bit_shift; +} -static inline unsigned long __xchg(unsigned long x, __volatile__ void * ptr, - int size) +static __always_inline unsigned long +__arch_xchg(unsigned long x, __volatile__ void * ptr, int size) { switch (size) { + case 2: + return xchg16(ptr, x); case 4: return xchg32(ptr, x); case 8: @@ -65,12 +110,11 @@ static inline unsigned long __xchg(unsigned long x, __volatile__ void * ptr, #include <asm-generic/cmpxchg-local.h> -#define __HAVE_ARCH_CMPXCHG 1 static inline unsigned long -__cmpxchg_u32(volatile int *m, int old, int new) +__cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new) { - __asm__ __volatile__("cas [%2], %3, %0" + __asm__ __volatile__("casx [%2], %3, %0" : "=&r" (new) : "0" (new), "r" (m), "r" (old) : "memory"); @@ -78,25 +122,43 @@ __cmpxchg_u32(volatile int *m, int old, int new) return new; } +/* + * Use 4 byte cas instruction to achieve 1 byte cmpxchg. Main logic + * here is to get the bit shift of the byte we are interested in. + * The XOR is handy for reversing the bits for big-endian byte order + */ static inline unsigned long -__cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new) +__cmpxchg_u8(volatile unsigned char *m, unsigned char old, unsigned char new) { - __asm__ __volatile__("casx [%2], %3, %0" - : "=&r" (new) - : "0" (new), "r" (m), "r" (old) - : "memory"); - - return new; + unsigned long maddr = (unsigned long)m; + int bit_shift = (((unsigned long)m & 3) ^ 3) << 3; + unsigned int mask = 0xff << bit_shift; + unsigned int *ptr = (unsigned int *) (maddr & ~3); + unsigned int old32, new32, load; + unsigned int load32 = *ptr; + + do { + new32 = (load32 & ~mask) | (new << bit_shift); + old32 = (load32 & ~mask) | (old << bit_shift); + load32 = __cmpxchg_u32(ptr, old32, new32); + if (load32 == old32) + return old; + load = (load32 & mask) >> bit_shift; + } while (load == old); + + return load; } /* This function doesn't exist, so you'll get a linker error if something tries to do an invalid cmpxchg(). */ -extern void __cmpxchg_called_with_bad_pointer(void); +void __cmpxchg_called_with_bad_pointer(void); static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) { switch (size) { + case 1: + return __cmpxchg_u8(ptr, old, new); case 4: return __cmpxchg_u32(ptr, old, new); case 8: @@ -106,7 +168,7 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) return old; } -#define cmpxchg(ptr,o,n) \ +#define arch_cmpxchg(ptr,o,n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ @@ -127,20 +189,20 @@ static inline unsigned long __cmpxchg_local(volatile void *ptr, case 4: case 8: return __cmpxchg(ptr, old, new, size); default: - return __cmpxchg_local_generic(ptr, old, new, size); + return __generic_cmpxchg_local(ptr, old, new, size); } return old; } -#define cmpxchg_local(ptr, o, n) \ +#define arch_cmpxchg_local(ptr, o, n) \ ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \ (unsigned long)(n), sizeof(*(ptr)))) -#define cmpxchg64_local(ptr, o, n) \ +#define arch_cmpxchg64_local(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_local((ptr), (o), (n)); \ + arch_cmpxchg_local((ptr), (o), (n)); \ }) -#define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) +#define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n)) #endif /* __ARCH_SPARC64_CMPXCHG__ */ diff --git a/arch/sparc/include/asm/compat.h b/arch/sparc/include/asm/compat.h index 830502fe62b4..e4382d2efa56 100644 --- a/arch/sparc/include/asm/compat.h +++ b/arch/sparc/include/asm/compat.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_SPARC64_COMPAT_H #define _ASM_SPARC64_COMPAT_H /* @@ -5,48 +6,28 @@ */ #include <linux/types.h> -#define COMPAT_USER_HZ 100 -#define COMPAT_UTS_MACHINE "sparc\0\0" +#define compat_mode_t compat_mode_t +typedef u16 compat_mode_t; -typedef u32 compat_size_t; -typedef s32 compat_ssize_t; -typedef s32 compat_time_t; -typedef s32 compat_clock_t; -typedef s32 compat_pid_t; +#define __compat_uid_t __compat_uid_t typedef u16 __compat_uid_t; typedef u16 __compat_gid_t; -typedef u32 __compat_uid32_t; -typedef u32 __compat_gid32_t; -typedef u16 compat_mode_t; -typedef u32 compat_ino_t; + +#define compat_dev_t compat_dev_t typedef u16 compat_dev_t; -typedef s32 compat_off_t; -typedef s64 compat_loff_t; -typedef s16 compat_nlink_t; -typedef u16 compat_ipc_pid_t; -typedef s32 compat_daddr_t; -typedef u32 compat_caddr_t; -typedef __kernel_fsid_t compat_fsid_t; -typedef s32 compat_key_t; -typedef s32 compat_timer_t; -typedef s32 compat_int_t; -typedef s32 compat_long_t; -typedef s64 compat_s64; -typedef u32 compat_uint_t; -typedef u32 compat_ulong_t; -typedef u64 compat_u64; -typedef u32 compat_uptr_t; +#define compat_ipc_pid_t compat_ipc_pid_t +typedef u16 compat_ipc_pid_t; -struct compat_timespec { - compat_time_t tv_sec; - s32 tv_nsec; -}; +#define compat_ipc64_perm compat_ipc64_perm -struct compat_timeval { - compat_time_t tv_sec; - s32 tv_usec; -}; +#define COMPAT_RLIM_INFINITY 0x7fffffff + +#include <asm-generic/compat.h> + +#define COMPAT_UTS_MACHINE "sparc\0\0" + +typedef s16 compat_nlink_t; struct compat_stat { compat_dev_t st_dev; @@ -57,11 +38,11 @@ struct compat_stat { __compat_gid_t st_gid; compat_dev_t st_rdev; compat_off_t st_size; - compat_time_t st_atime; + old_time32_t st_atime; compat_ulong_t st_atime_nsec; - compat_time_t st_mtime; + old_time32_t st_mtime; compat_ulong_t st_mtime_nsec; - compat_time_t st_ctime; + old_time32_t st_ctime; compat_ulong_t st_ctime_nsec; compat_off_t st_blksize; compat_off_t st_blocks; @@ -102,147 +83,7 @@ struct compat_stat64 { unsigned int __unused5; }; -struct compat_flock { - short l_type; - short l_whence; - compat_off_t l_start; - compat_off_t l_len; - compat_pid_t l_pid; - short __unused; -}; - -#define F_GETLK64 12 -#define F_SETLK64 13 -#define F_SETLKW64 14 - -struct compat_flock64 { - short l_type; - short l_whence; - compat_loff_t l_start; - compat_loff_t l_len; - compat_pid_t l_pid; - short __unused; -}; - -struct compat_statfs { - int f_type; - int f_bsize; - int f_blocks; - int f_bfree; - int f_bavail; - int f_files; - int f_ffree; - compat_fsid_t f_fsid; - int f_namelen; /* SunOS ignores this field. */ - int f_frsize; - int f_flags; - int f_spare[4]; -}; - -#define COMPAT_RLIM_INFINITY 0x7fffffff - -typedef u32 compat_old_sigset_t; - -#define _COMPAT_NSIG 64 -#define _COMPAT_NSIG_BPW 32 - -typedef u32 compat_sigset_word; - -typedef union compat_sigval { - compat_int_t sival_int; - compat_uptr_t sival_ptr; -} compat_sigval_t; - -#define SI_PAD_SIZE32 (128/sizeof(int) - 3) - -typedef struct compat_siginfo { - int si_signo; - int si_errno; - int si_code; - - union { - int _pad[SI_PAD_SIZE32]; - - /* kill() */ - struct { - compat_pid_t _pid; /* sender's pid */ - unsigned int _uid; /* sender's uid */ - } _kill; - - /* POSIX.1b timers */ - struct { - compat_timer_t _tid; /* timer id */ - int _overrun; /* overrun count */ - compat_sigval_t _sigval; /* same as below */ - int _sys_private; /* not to be passed to user */ - } _timer; - - /* POSIX.1b signals */ - struct { - compat_pid_t _pid; /* sender's pid */ - unsigned int _uid; /* sender's uid */ - compat_sigval_t _sigval; - } _rt; - - /* SIGCHLD */ - struct { - compat_pid_t _pid; /* which child */ - unsigned int _uid; /* sender's uid */ - int _status; /* exit code */ - compat_clock_t _utime; - compat_clock_t _stime; - } _sigchld; - - /* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */ - struct { - u32 _addr; /* faulting insn/memory ref. */ - int _trapno; - } _sigfault; - - /* SIGPOLL */ - struct { - int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ - int _fd; - } _sigpoll; - } _sifields; -} compat_siginfo_t; - -#define COMPAT_OFF_T_MAX 0x7fffffff -#define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL - -/* - * A pointer passed in from user mode. This should not - * be used for syscall parameters, just declare them - * as pointers because the syscall entry code will have - * appropriately converted them already. - */ - -static inline void __user *compat_ptr(compat_uptr_t uptr) -{ - return (void __user *)(unsigned long)uptr; -} - -static inline compat_uptr_t ptr_to_compat(void __user *uptr) -{ - return (u32)(unsigned long)uptr; -} - -static inline void __user *arch_compat_alloc_user_space(long len) -{ - struct pt_regs *regs = current_thread_info()->kregs; - unsigned long usp = regs->u_regs[UREG_I6]; - - if (test_thread_64bit_stack(usp)) - usp += STACK_BIAS; - - if (test_thread_flag(TIF_32BIT)) - usp &= 0xffffffffUL; - - usp -= len; - usp &= ~0x7UL; - - return (void __user *) usp; -} +#define __ARCH_COMPAT_FLOCK_PAD short __unused; struct compat_ipc64_perm { compat_key_t key; @@ -260,10 +101,10 @@ struct compat_ipc64_perm { struct compat_semid64_ds { struct compat_ipc64_perm sem_perm; - unsigned int __pad1; - compat_time_t sem_otime; - unsigned int __pad2; - compat_time_t sem_ctime; + unsigned int sem_otime_high; + unsigned int sem_otime; + unsigned int sem_ctime_high; + unsigned int sem_ctime; u32 sem_nsems; u32 __unused1; u32 __unused2; @@ -271,12 +112,12 @@ struct compat_semid64_ds { struct compat_msqid64_ds { struct compat_ipc64_perm msg_perm; - unsigned int __pad1; - compat_time_t msg_stime; - unsigned int __pad2; - compat_time_t msg_rtime; - unsigned int __pad3; - compat_time_t msg_ctime; + unsigned int msg_stime_high; + unsigned int msg_stime; + unsigned int msg_rtime_high; + unsigned int msg_rtime; + unsigned int msg_ctime_high; + unsigned int msg_ctime; unsigned int msg_cbytes; unsigned int msg_qnum; unsigned int msg_qbytes; @@ -288,12 +129,12 @@ struct compat_msqid64_ds { struct compat_shmid64_ds { struct compat_ipc64_perm shm_perm; - unsigned int __pad1; - compat_time_t shm_atime; - unsigned int __pad2; - compat_time_t shm_dtime; - unsigned int __pad3; - compat_time_t shm_ctime; + unsigned int shm_atime_high; + unsigned int shm_atime; + unsigned int shm_dtime_high; + unsigned int shm_dtime; + unsigned int shm_ctime_high; + unsigned int shm_ctime; compat_size_t shm_segsz; compat_pid_t shm_cpid; compat_pid_t shm_lpid; @@ -302,9 +143,18 @@ struct compat_shmid64_ds { unsigned int __unused2; }; +#ifdef CONFIG_COMPAT static inline int is_compat_task(void) { return test_thread_flag(TIF_32BIT); } +static inline bool in_compat_syscall(void) +{ + /* Vector 0x110 is LINUX_32BIT_SYSCALL_TRAP */ + return pt_regs_trap_type(current_pt_regs()) == 0x110; +} +#define in_compat_syscall in_compat_syscall +#endif + #endif /* _ASM_SPARC64_COMPAT_H */ diff --git a/arch/sparc/include/asm/compat_signal.h b/arch/sparc/include/asm/compat_signal.h index 9ed1f128b4d1..e5f7a7f281d2 100644 --- a/arch/sparc/include/asm/compat_signal.h +++ b/arch/sparc/include/asm/compat_signal.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _COMPAT_SIGNAL_H #define _COMPAT_SIGNAL_H @@ -6,17 +7,17 @@ #ifdef CONFIG_COMPAT struct __new_sigaction32 { - unsigned sa_handler; + unsigned int sa_handler; unsigned int sa_flags; - unsigned sa_restorer; /* not used by Linux/SPARC yet */ + unsigned int sa_restorer; /* not used by Linux/SPARC yet */ compat_sigset_t sa_mask; }; struct __old_sigaction32 { - unsigned sa_handler; + unsigned int sa_handler; compat_old_sigset_t sa_mask; unsigned int sa_flags; - unsigned sa_restorer; /* not used by Linux/SPARC yet */ + unsigned int sa_restorer; /* not used by Linux/SPARC yet */ }; #endif diff --git a/arch/sparc/include/asm/contregs.h b/arch/sparc/include/asm/contregs.h index b8abdfcf5555..4df56a6ba7d9 100644 --- a/arch/sparc/include/asm/contregs.h +++ b/arch/sparc/include/asm/contregs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_CONTREGS_H #define _SPARC_CONTREGS_H diff --git a/arch/sparc/include/asm/cpu_type.h b/arch/sparc/include/asm/cpu_type.h index 84d7d83b8084..2b59799859d1 100644 --- a/arch/sparc/include/asm/cpu_type.h +++ b/arch/sparc/include/asm/cpu_type.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASM_CPU_TYPE_H #define __ASM_CPU_TYPE_H diff --git a/arch/sparc/include/asm/cpudata.h b/arch/sparc/include/asm/cpudata.h index b5976de7cacd..67022a153023 100644 --- a/arch/sparc/include/asm/cpudata.h +++ b/arch/sparc/include/asm/cpudata.h @@ -1,5 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_CPUDATA_H #define ___ASM_SPARC_CPUDATA_H + +#ifndef __ASSEMBLER__ + +#include <linux/threads.h> +#include <linux/percpu.h> + +extern const struct seq_operations cpuinfo_op; + +#endif /* !(__ASSEMBLER__) */ + #if defined(__sparc__) && defined(__arch64__) #include <asm/cpudata_64.h> #else diff --git a/arch/sparc/include/asm/cpudata_32.h b/arch/sparc/include/asm/cpudata_32.h index 0300d94c25b3..895d0964b802 100644 --- a/arch/sparc/include/asm/cpudata_32.h +++ b/arch/sparc/include/asm/cpudata_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* cpudata.h: Per-cpu parameters. * * Copyright (C) 2004 Keith M Wesolowski (wesolows@foobazco.org) @@ -26,6 +27,6 @@ typedef struct { DECLARE_PER_CPU(cpuinfo_sparc, __cpu_data); #define cpu_data(__cpu) per_cpu(__cpu_data, (__cpu)) -#define local_cpu_data() __get_cpu_var(__cpu_data) +#define local_cpu_data() (*this_cpu_ptr(&__cpu_data)) #endif /* _SPARC_CPUDATA_H */ diff --git a/arch/sparc/include/asm/cpudata_64.h b/arch/sparc/include/asm/cpudata_64.h index 050ef35b9dcf..056b3c0e7ef9 100644 --- a/arch/sparc/include/asm/cpudata_64.h +++ b/arch/sparc/include/asm/cpudata_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* cpudata.h: Per-cpu parameters. * * Copyright (C) 2003, 2005, 2006 David S. Miller (davem@davemloft.net) @@ -6,10 +7,7 @@ #ifndef _SPARC64_CPUDATA_H #define _SPARC64_CPUDATA_H -#ifndef __ASSEMBLY__ - -#include <linux/percpu.h> -#include <linux/threads.h> +#ifndef __ASSEMBLER__ typedef struct { /* Dcache line 1 */ @@ -27,17 +25,17 @@ typedef struct { unsigned int icache_line_size; unsigned int ecache_size; unsigned int ecache_line_size; - int core_id; - int proc_id; + unsigned short sock_id; /* physical package */ + unsigned short core_id; + unsigned short max_cache_id; /* groupings of highest shared cache */ + signed short proc_id; /* strand (aka HW thread) id */ } cpuinfo_sparc; DECLARE_PER_CPU(cpuinfo_sparc, __cpu_data); #define cpu_data(__cpu) per_cpu(__cpu_data, (__cpu)) -#define local_cpu_data() __get_cpu_var(__cpu_data) - -extern const struct seq_operations cpuinfo_op; +#define local_cpu_data() (*this_cpu_ptr(&__cpu_data)) -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #include <asm/trap_block.h> diff --git a/arch/sparc/include/asm/current.h b/arch/sparc/include/asm/current.h index 10a0df55a574..c68312a89d14 100644 --- a/arch/sparc/include/asm/current.h +++ b/arch/sparc/include/asm/current.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* include/asm/current.h * * Copyright (C) 1999 IBM Deutschland Entwicklung GmbH, IBM Corporation diff --git a/arch/sparc/include/asm/dcr.h b/arch/sparc/include/asm/dcr.h index 620c9ba642e9..c810e1b4285c 100644 --- a/arch/sparc/include/asm/dcr.h +++ b/arch/sparc/include/asm/dcr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_DCR_H #define _SPARC64_DCR_H diff --git a/arch/sparc/include/asm/dcu.h b/arch/sparc/include/asm/dcu.h index 0f704e106a1b..93f32507125e 100644 --- a/arch/sparc/include/asm/dcu.h +++ b/arch/sparc/include/asm/dcu.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_DCU_H #define _SPARC64_DCU_H diff --git a/arch/sparc/include/asm/delay.h b/arch/sparc/include/asm/delay.h index 467caa2a97a0..c96af9b91661 100644 --- a/arch/sparc/include/asm/delay.h +++ b/arch/sparc/include/asm/delay.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_DELAY_H #define ___ASM_SPARC_DELAY_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/delay_32.h b/arch/sparc/include/asm/delay_32.h index bc9aba2bead6..0e6dfe857d67 100644 --- a/arch/sparc/include/asm/delay_32.h +++ b/arch/sparc/include/asm/delay_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * delay.h: Linux delay routines on the Sparc. * @@ -20,8 +21,8 @@ static inline void __delay(unsigned long loops) } /* This is too messy with inline asm on the Sparc. */ -extern void __udelay(unsigned long usecs, unsigned long lpj); -extern void __ndelay(unsigned long nsecs, unsigned long lpj); +void __udelay(unsigned long usecs, unsigned long lpj); +void __ndelay(unsigned long nsecs, unsigned long lpj); #ifdef CONFIG_SMP #define __udelay_val cpu_data(smp_processor_id()).udelay_val diff --git a/arch/sparc/include/asm/delay_64.h b/arch/sparc/include/asm/delay_64.h index a77aa622d762..5de5b5f23188 100644 --- a/arch/sparc/include/asm/delay_64.h +++ b/arch/sparc/include/asm/delay_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* delay.h: Linux delay routines on sparc64. * * Copyright (C) 1996, 2004, 2007 David S. Miller (davem@davemloft.net). @@ -6,12 +7,12 @@ #ifndef _SPARC64_DELAY_H #define _SPARC64_DELAY_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ -extern void __delay(unsigned long loops); -extern void udelay(unsigned long usecs); +void __delay(unsigned long loops); +void udelay(unsigned long usecs); #define mdelay(n) udelay((n) * 1000) -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* _SPARC64_DELAY_H */ diff --git a/arch/sparc/include/asm/device.h b/arch/sparc/include/asm/device.h index daa6a8a5e9cd..a797d5e86406 100644 --- a/arch/sparc/include/asm/device.h +++ b/arch/sparc/include/asm/device.h @@ -1,7 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Arch specific extensions to struct device - * - * This file is released under the GPLv2 */ #ifndef _ASM_SPARC_DEVICE_H #define _ASM_SPARC_DEVICE_H @@ -19,7 +18,7 @@ struct dev_archdata { int numa_node; }; -extern void of_propagate_archdata(struct platform_device *bus); +void of_propagate_archdata(struct platform_device *bus); struct pdev_archdata { struct resource resource[PROMREG_MAX]; diff --git a/arch/sparc/include/asm/dma-mapping.h b/arch/sparc/include/asm/dma-mapping.h index 05fe53f5346e..55c12fc2ba63 100644 --- a/arch/sparc/include/asm/dma-mapping.h +++ b/arch/sparc/include/asm/dma-mapping.h @@ -1,79 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_DMA_MAPPING_H #define ___ASM_SPARC_DMA_MAPPING_H -#include <linux/scatterlist.h> -#include <linux/mm.h> -#include <linux/dma-debug.h> +extern const struct dma_map_ops *dma_ops; -#define DMA_ERROR_CODE (~(dma_addr_t)0x0) - -extern int dma_supported(struct device *dev, u64 mask); - -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - -extern struct dma_map_ops *dma_ops; -extern struct dma_map_ops *leon_dma_ops; -extern struct dma_map_ops pci32_dma_ops; - -extern struct bus_type pci_bus_type; - -static inline struct dma_map_ops *get_dma_ops(struct device *dev) -{ -#if defined(CONFIG_SPARC32) && defined(CONFIG_PCI) - if (sparc_cpu_model == sparc_leon) - return leon_dma_ops; - else if (dev->bus == &pci_bus_type) - return &pci32_dma_ops; -#endif - return dma_ops; -} - -#include <asm-generic/dma-mapping-common.h> - -#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - void *cpu_addr; - - cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs); - debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr); - return cpu_addr; -} - -#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); - ops->free(dev, size, cpu_addr, dma_handle, attrs); -} - -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - debug_dma_mapping_error(dev, dma_addr); - return (dma_addr == DMA_ERROR_CODE); -} - -static inline int dma_set_mask(struct device *dev, u64 mask) +static inline const struct dma_map_ops *get_arch_dma_ops(void) { -#ifdef CONFIG_PCI - if (dev->bus == &pci_bus_type) { - if (!dev->dma_mask || !dma_supported(dev, mask)) - return -EINVAL; - *dev->dma_mask = mask; - return 0; - } -#endif - return -EINVAL; + /* sparc32 uses per-device dma_ops */ + return IS_ENABLED(CONFIG_SPARC64) ? dma_ops : NULL; } #endif diff --git a/arch/sparc/include/asm/dma.h b/arch/sparc/include/asm/dma.h index 3d434ef5eae3..08043f35b110 100644 --- a/arch/sparc/include/asm/dma.h +++ b/arch/sparc/include/asm/dma.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_SPARC_DMA_H #define _ASM_SPARC_DMA_H @@ -81,63 +82,11 @@ #define DMA_BURST64 0x40 #define DMA_BURSTBITS 0x7f -/* From PCI */ - -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - #ifdef CONFIG_SPARC32 - -/* Routines for data transfer buffers. */ struct device; -struct scatterlist; - -struct sparc32_dma_ops { - __u32 (*get_scsi_one)(struct device *, char *, unsigned long); - void (*get_scsi_sgl)(struct device *, struct scatterlist *, int); - void (*release_scsi_one)(struct device *, __u32, unsigned long); - void (*release_scsi_sgl)(struct device *, struct scatterlist *,int); -#ifdef CONFIG_SBUS - int (*map_dma_area)(struct device *, dma_addr_t *, unsigned long, unsigned long, int); - void (*unmap_dma_area)(struct device *, unsigned long, int); -#endif -}; -extern const struct sparc32_dma_ops *sparc32_dma_ops; - -#define mmu_get_scsi_one(dev,vaddr,len) \ - sparc32_dma_ops->get_scsi_one(dev, vaddr, len) -#define mmu_get_scsi_sgl(dev,sg,sz) \ - sparc32_dma_ops->get_scsi_sgl(dev, sg, sz) -#define mmu_release_scsi_one(dev,vaddr,len) \ - sparc32_dma_ops->release_scsi_one(dev, vaddr,len) -#define mmu_release_scsi_sgl(dev,sg,sz) \ - sparc32_dma_ops->release_scsi_sgl(dev, sg, sz) - -#ifdef CONFIG_SBUS -/* - * mmu_map/unmap are provided by iommu/iounit; Invalid to call on IIep. - * - * The mmu_map_dma_area establishes two mappings in one go. - * These mappings point to pages normally mapped at 'va' (linear address). - * First mapping is for CPU visible address at 'a', uncached. - * This is an alias, but it works because it is an uncached mapping. - * Second mapping is for device visible address, or "bus" address. - * The bus address is returned at '*pba'. - * - * These functions seem distinct, but are hard to split. - * On sun4m, page attributes depend on the CPU type, so we have to - * know if we are mapping RAM or I/O, so it has to be an additional argument - * to a separate mapping function for CPU visible mappings. - */ -#define sbus_map_dma_area(dev,pba,va,a,len) \ - sparc32_dma_ops->map_dma_area(dev, pba, va, a, len) -#define sbus_unmap_dma_area(dev,ba,len) \ - sparc32_dma_ops->unmap_dma_area(dev, ba, len) -#endif /* CONFIG_SBUS */ +unsigned long sparc_dma_alloc_resource(struct device *dev, size_t len); +bool sparc_dma_free_resource(void *cpu_addr, size_t size); #endif #endif /* !(_ASM_SPARC_DMA_H) */ diff --git a/arch/sparc/include/asm/ebus_dma.h b/arch/sparc/include/asm/ebus_dma.h index f07a5b541c98..75563ed090f5 100644 --- a/arch/sparc/include/asm/ebus_dma.h +++ b/arch/sparc/include/asm/ebus_dma.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASM_SPARC_EBUS_DMA_H #define __ASM_SPARC_EBUS_DMA_H @@ -22,14 +23,14 @@ struct ebus_dma_info { unsigned char name[64]; }; -extern int ebus_dma_register(struct ebus_dma_info *p); -extern int ebus_dma_irq_enable(struct ebus_dma_info *p, int on); -extern void ebus_dma_unregister(struct ebus_dma_info *p); -extern int ebus_dma_request(struct ebus_dma_info *p, dma_addr_t bus_addr, +int ebus_dma_register(struct ebus_dma_info *p); +int ebus_dma_irq_enable(struct ebus_dma_info *p, int on); +void ebus_dma_unregister(struct ebus_dma_info *p); +int ebus_dma_request(struct ebus_dma_info *p, dma_addr_t bus_addr, size_t len); -extern void ebus_dma_prepare(struct ebus_dma_info *p, int write); -extern unsigned int ebus_dma_residue(struct ebus_dma_info *p); -extern unsigned int ebus_dma_addr(struct ebus_dma_info *p); -extern void ebus_dma_enable(struct ebus_dma_info *p, int on); +void ebus_dma_prepare(struct ebus_dma_info *p, int write); +unsigned int ebus_dma_residue(struct ebus_dma_info *p); +unsigned int ebus_dma_addr(struct ebus_dma_info *p); +void ebus_dma_enable(struct ebus_dma_info *p, int on); #endif /* __ASM_SPARC_EBUS_DMA_H */ diff --git a/arch/sparc/include/asm/ecc.h b/arch/sparc/include/asm/ecc.h index ccb84b66fef1..3d5edee36b4c 100644 --- a/arch/sparc/include/asm/ecc.h +++ b/arch/sparc/include/asm/ecc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * ecc.h: Definitions and defines for the external cache/memory * controller on the sun4m. diff --git a/arch/sparc/include/asm/elf.h b/arch/sparc/include/asm/elf.h index 0a2816c50b07..bbfb4b002ff3 100644 --- a/arch/sparc/include/asm/elf.h +++ b/arch/sparc/include/asm/elf.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_ELF_H #define ___ASM_SPARC_ELF_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/elf_32.h b/arch/sparc/include/asm/elf_32.h index a24e41fcdde1..37a6016c9ccd 100644 --- a/arch/sparc/include/asm/elf_32.h +++ b/arch/sparc/include/asm/elf_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASMSPARC_ELF_H #define __ASMSPARC_ELF_H diff --git a/arch/sparc/include/asm/elf_64.h b/arch/sparc/include/asm/elf_64.h index 370ca1e71ffb..694ed081cf8d 100644 --- a/arch/sparc/include/asm/elf_64.h +++ b/arch/sparc/include/asm/elf_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASM_SPARC64_ELF_H #define __ASM_SPARC64_ELF_H @@ -7,8 +8,8 @@ #include <asm/ptrace.h> #include <asm/processor.h> -#include <asm/uaccess.h> #include <asm/spitfire.h> +#include <asm/adi.h> /* * Sparc section types @@ -57,6 +58,7 @@ #define R_SPARC_7 43 #define R_SPARC_5 44 #define R_SPARC_6 45 +#define R_SPARC_UA64 54 /* Bits present in AT_HWCAP, primarily for Sparc32. */ #define HWCAP_SPARC_FLUSH 0x00000001 @@ -95,6 +97,7 @@ * really available. So we simply advertise only "crypto" support. */ #define HWCAP_SPARC_CRYPTO 0x04000000 /* CRYPTO insns available */ +#define HWCAP_SPARC_ADI 0x08000000 /* ADI available */ #define CORE_DUMP_USE_REGSET @@ -209,4 +212,22 @@ do { if ((ex).e_ident[EI_CLASS] == ELFCLASS32) \ (current->personality & (~PER_MASK))); \ } while (0) +extern unsigned int vdso_enabled; + +#define ARCH_DLINFO \ +do { \ + extern struct adi_config adi_state; \ + if (vdso_enabled) \ + NEW_AUX_ENT(AT_SYSINFO_EHDR, \ + (unsigned long)current->mm->context.vdso); \ + NEW_AUX_ENT(AT_ADI_BLKSZ, adi_state.caps.blksz); \ + NEW_AUX_ENT(AT_ADI_NBITS, adi_state.caps.nbits); \ + NEW_AUX_ENT(AT_ADI_UEONADI, adi_state.caps.ue_on_adi); \ +} while (0) + +struct linux_binprm; + +#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1 +extern int arch_setup_additional_pages(struct linux_binprm *bprm, + int uses_interp); #endif /* !(__ASM_SPARC64_ELF_H) */ diff --git a/arch/sparc/include/asm/estate.h b/arch/sparc/include/asm/estate.h index 520c08560d1b..e5e0f84accea 100644 --- a/arch/sparc/include/asm/estate.h +++ b/arch/sparc/include/asm/estate.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_ESTATE_H #define _SPARC64_ESTATE_H diff --git a/arch/sparc/include/asm/extable.h b/arch/sparc/include/asm/extable.h new file mode 100644 index 000000000000..554a9dc376fc --- /dev/null +++ b/arch/sparc/include/asm/extable.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_EXTABLE_H +#define __ASM_EXTABLE_H +/* + * The exception table consists of pairs of addresses: the first is the + * address of an instruction that is allowed to fault, and the second is + * the address at which the program should continue. No registers are + * modified, so it is entirely up to the continuation code to figure out + * what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ + +struct exception_table_entry { + unsigned int insn, fixup; +}; + +#endif diff --git a/arch/sparc/include/asm/fb.h b/arch/sparc/include/asm/fb.h deleted file mode 100644 index 2173432ad7f7..000000000000 --- a/arch/sparc/include/asm/fb.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef _SPARC_FB_H_ -#define _SPARC_FB_H_ -#include <linux/console.h> -#include <linux/fb.h> -#include <linux/fs.h> -#include <asm/page.h> -#include <asm/prom.h> - -static inline void fb_pgprotect(struct file *file, struct vm_area_struct *vma, - unsigned long off) -{ -#ifdef CONFIG_SPARC64 - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); -#endif -} - -static inline int fb_is_primary_device(struct fb_info *info) -{ - struct device *dev = info->device; - struct device_node *node; - - if (console_set_on_cmdline) - return 0; - - node = dev->of_node; - if (node && - node == of_console_device) - return 1; - - return 0; -} - -#endif /* _SPARC_FB_H_ */ diff --git a/arch/sparc/include/asm/fbio.h b/arch/sparc/include/asm/fbio.h index 1d9afe277e9c..02654cb95dec 100644 --- a/arch/sparc/include/asm/fbio.h +++ b/arch/sparc/include/asm/fbio.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __LINUX_FBIO_H #define __LINUX_FBIO_H diff --git a/arch/sparc/include/asm/fhc.h b/arch/sparc/include/asm/fhc.h index 57f1b303ad54..0627fa126540 100644 --- a/arch/sparc/include/asm/fhc.h +++ b/arch/sparc/include/asm/fhc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* fhc.h: FHC and Clock board register definitions. * * Copyright (C) 1997, 1999 David S. Miller (davem@redhat.com) diff --git a/arch/sparc/include/asm/floppy.h b/arch/sparc/include/asm/floppy.h index faebd335b600..4b315802e635 100644 --- a/arch/sparc/include/asm/floppy.h +++ b/arch/sparc/include/asm/floppy.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_FLOPPY_H #define ___ASM_SPARC_FLOPPY_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/floppy_32.h b/arch/sparc/include/asm/floppy_32.h index fb3f16954c69..7251d1fed7a4 100644 --- a/arch/sparc/include/asm/floppy_32.h +++ b/arch/sparc/include/asm/floppy_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* asm/floppy.h: Sparc specific parts of the Floppy driver. * * Copyright (C) 1995 David S. Miller (davem@davemloft.net) @@ -7,13 +8,14 @@ #define __ASM_SPARC_FLOPPY_H #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/of_platform.h> +#include <linux/pgtable.h> -#include <asm/page.h> -#include <asm/pgtable.h> #include <asm/idprom.h> #include <asm/oplib.h> #include <asm/auxio.h> +#include <asm/setup.h> +#include <asm/page.h> #include <asm/irq.h> /* We don't need no stinkin' I/O port allocation crap. */ @@ -49,7 +51,6 @@ struct sun_flpy_controller { /* You'll only ever find one controller on a SparcStation anyways. */ static struct sun_flpy_controller *sun_fdc = NULL; -extern volatile unsigned char *fdc_status; struct sun_floppy_ops { unsigned char (*fd_inb)(int port); @@ -58,8 +59,8 @@ struct sun_floppy_ops { static struct sun_floppy_ops sun_fdops; -#define fd_inb(port) sun_fdops.fd_inb(port) -#define fd_outb(value,port) sun_fdops.fd_outb(value,port) +#define fd_inb(base, reg) sun_fdops.fd_inb(reg) +#define fd_outb(value, base, reg) sun_fdops.fd_outb(value, reg) #define fd_enable_dma() sun_fd_enable_dma() #define fd_disable_dma() sun_fd_disable_dma() #define fd_request_dma() (0) /* nothing... */ @@ -70,7 +71,6 @@ static struct sun_floppy_ops sun_fdops; #define fd_set_dma_count(count) sun_fd_set_dma_count(count) #define fd_enable_irq() /* nothing... */ #define fd_disable_irq() /* nothing... */ -#define fd_cacheflush(addr, size) /* nothing... */ #define fd_request_irq() sun_fd_request_irq() #define fd_free_irq() /* nothing... */ #if 0 /* P3: added by Alain, these cause a MMU corruption. 19960524 XXX */ @@ -96,9 +96,6 @@ static struct sun_floppy_ops sun_fdops; #define N_FDC 1 #define N_DRIVE 8 -/* No 64k boundary crossing problems on the Sparc. */ -#define CROSS_64KB(a,s) (0) - /* Routines unique to each controller type on a Sun. */ static void sun_set_dor(unsigned char value, int fdc_82077) { @@ -114,15 +111,15 @@ static unsigned char sun_read_dir(void) static unsigned char sun_82072_fd_inb(int port) { udelay(5); - switch(port & 7) { + switch (port) { default: printk("floppy: Asked to read unknown port %d\n", port); panic("floppy: Port bolixed."); - case 4: /* FD_STATUS */ + case FD_STATUS: return sun_fdc->status_82072 & ~STATUS_DMA; - case 5: /* FD_DATA */ + case FD_DATA: return sun_fdc->data_82072; - case 7: /* FD_DIR */ + case FD_DIR: return sun_read_dir(); } panic("sun_82072_fd_inb: How did I get here?"); @@ -131,20 +128,20 @@ static unsigned char sun_82072_fd_inb(int port) static void sun_82072_fd_outb(unsigned char value, int port) { udelay(5); - switch(port & 7) { + switch (port) { default: printk("floppy: Asked to write to unknown port %d\n", port); panic("floppy: Port bolixed."); - case 2: /* FD_DOR */ + case FD_DOR: sun_set_dor(value, 0); break; - case 5: /* FD_DATA */ + case FD_DATA: sun_fdc->data_82072 = value; break; - case 7: /* FD_DCR */ + case FD_DCR: sun_fdc->dcr_82072 = value; break; - case 4: /* FD_STATUS */ + case FD_DSR: sun_fdc->status_82072 = value; break; } @@ -154,23 +151,23 @@ static void sun_82072_fd_outb(unsigned char value, int port) static unsigned char sun_82077_fd_inb(int port) { udelay(5); - switch(port & 7) { + switch (port) { default: printk("floppy: Asked to read unknown port %d\n", port); panic("floppy: Port bolixed."); - case 0: /* FD_STATUS_0 */ + case FD_SRA: return sun_fdc->status1_82077; - case 1: /* FD_STATUS_1 */ + case FD_SRB: return sun_fdc->status2_82077; - case 2: /* FD_DOR */ + case FD_DOR: return sun_fdc->dor_82077; - case 3: /* FD_TDR */ + case FD_TDR: return sun_fdc->tapectl_82077; - case 4: /* FD_STATUS */ + case FD_STATUS: return sun_fdc->status_82077 & ~STATUS_DMA; - case 5: /* FD_DATA */ + case FD_DATA: return sun_fdc->data_82077; - case 7: /* FD_DIR */ + case FD_DIR: return sun_read_dir(); } panic("sun_82077_fd_inb: How did I get here?"); @@ -179,23 +176,23 @@ static unsigned char sun_82077_fd_inb(int port) static void sun_82077_fd_outb(unsigned char value, int port) { udelay(5); - switch(port & 7) { + switch (port) { default: printk("floppy: Asked to write to unknown port %d\n", port); panic("floppy: Port bolixed."); - case 2: /* FD_DOR */ + case FD_DOR: sun_set_dor(value, 1); break; - case 5: /* FD_DATA */ + case FD_DATA: sun_fdc->data_82077 = value; break; - case 7: /* FD_DCR */ + case FD_DCR: sun_fdc->dcr_82077 = value; break; - case 4: /* FD_STATUS */ + case FD_DSR: sun_fdc->status_82077 = value; break; - case 3: /* FD_TDR */ + case FD_TDR: sun_fdc->tapectl_82077 = value; break; } @@ -212,13 +209,6 @@ static void sun_82077_fd_outb(unsigned char value, int port) * underruns. If non-zero, doing_pdma encodes the direction of * the transfer for debugging. 1=read 2=write */ -extern char *pdma_vaddr; -extern unsigned long pdma_size; -extern volatile int doing_pdma; - -/* This is software state */ -extern char *pdma_base; -extern unsigned long pdma_areasize; /* Common routines to all controller types on the Sparc. */ static inline void virtual_dma_init(void) @@ -263,8 +253,7 @@ static inline void sun_fd_enable_dma(void) pdma_areasize = pdma_size; } -extern int sparc_floppy_request_irq(unsigned int irq, - irq_handler_t irq_handler); +int sparc_floppy_request_irq(unsigned int irq, irq_handler_t irq_handler); static int sun_fd_request_irq(void) { diff --git a/arch/sparc/include/asm/floppy_64.h b/arch/sparc/include/asm/floppy_64.h index e204f902e6c9..d1bb0f13352c 100644 --- a/arch/sparc/include/asm/floppy_64.h +++ b/arch/sparc/include/asm/floppy_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* floppy.h: Sparc specific parts of the Floppy driver. * * Copyright (C) 1996, 2007, 2008 David S. Miller (davem@davemloft.net) @@ -10,8 +11,9 @@ #define __ASM_SPARC64_FLOPPY_H #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/of_platform.h> #include <linux/dma-mapping.h> +#include <linux/string.h> #include <asm/auxio.h> @@ -46,8 +48,9 @@ unsigned long fdc_status; static struct platform_device *floppy_op = NULL; struct sun_floppy_ops { - unsigned char (*fd_inb) (unsigned long port); - void (*fd_outb) (unsigned char value, unsigned long port); + unsigned char (*fd_inb) (unsigned long port, unsigned int reg); + void (*fd_outb) (unsigned char value, unsigned long base, + unsigned int reg); void (*fd_enable_dma) (void); void (*fd_disable_dma) (void); void (*fd_set_dma_mode) (int); @@ -61,8 +64,8 @@ struct sun_floppy_ops { static struct sun_floppy_ops sun_fdops; -#define fd_inb(port) sun_fdops.fd_inb(port) -#define fd_outb(value,port) sun_fdops.fd_outb(value,port) +#define fd_inb(base, reg) sun_fdops.fd_inb(base, reg) +#define fd_outb(value, base, reg) sun_fdops.fd_outb(value, base, reg) #define fd_enable_dma() sun_fdops.fd_enable_dma() #define fd_disable_dma() sun_fdops.fd_disable_dma() #define fd_request_dma() (0) /* nothing... */ @@ -72,7 +75,6 @@ static struct sun_floppy_ops sun_fdops; #define fd_set_dma_addr(addr) sun_fdops.fd_set_dma_addr(addr) #define fd_set_dma_count(count) sun_fdops.fd_set_dma_count(count) #define get_dma_residue(x) sun_fdops.get_dma_residue() -#define fd_cacheflush(addr, size) /* nothing... */ #define fd_request_irq() sun_fdops.fd_request_irq() #define fd_free_irq() sun_fdops.fd_free_irq() #define fd_eject(drive) sun_fdops.fd_eject(drive) @@ -94,45 +96,43 @@ static int sun_floppy_types[2] = { 0, 0 }; #define N_FDC 1 #define N_DRIVE 8 -/* No 64k boundary crossing problems on the Sparc. */ -#define CROSS_64KB(a,s) (0) - -static unsigned char sun_82077_fd_inb(unsigned long port) +static unsigned char sun_82077_fd_inb(unsigned long base, unsigned int reg) { udelay(5); - switch(port & 7) { + switch (reg) { default: - printk("floppy: Asked to read unknown port %lx\n", port); + printk("floppy: Asked to read unknown port %x\n", reg); panic("floppy: Port bolixed."); - case 4: /* FD_STATUS */ + case FD_STATUS: return sbus_readb(&sun_fdc->status_82077) & ~STATUS_DMA; - case 5: /* FD_DATA */ + case FD_DATA: return sbus_readb(&sun_fdc->data_82077); - case 7: /* FD_DIR */ + case FD_DIR: /* XXX: Is DCL on 0x80 in sun4m? */ return sbus_readb(&sun_fdc->dir_82077); } panic("sun_82072_fd_inb: How did I get here?"); } -static void sun_82077_fd_outb(unsigned char value, unsigned long port) +static void sun_82077_fd_outb(unsigned char value, unsigned long base, + unsigned int reg) { udelay(5); - switch(port & 7) { + switch (reg) { default: - printk("floppy: Asked to write to unknown port %lx\n", port); + printk("floppy: Asked to write to unknown port %x\n", reg); panic("floppy: Port bolixed."); - case 2: /* FD_DOR */ + case FD_DOR: /* Happily, the 82077 has a real DOR register. */ sbus_writeb(value, &sun_fdc->dor_82077); break; - case 5: /* FD_DATA */ + case FD_DATA: sbus_writeb(value, &sun_fdc->data_82077); break; - case 7: /* FD_DCR */ + case FD_DCR: sbus_writeb(value, &sun_fdc->dcr_82077); break; - case 4: /* FD_STATUS */ + case FD_DSR: sbus_writeb(value, &sun_fdc->status_82077); break; } @@ -195,7 +195,7 @@ static void sun_fd_enable_dma(void) pdma_areasize = pdma_size; } -irqreturn_t sparc_floppy_irq(int irq, void *dev_cookie) +static irqreturn_t sparc_floppy_irq(int irq, void *dev_cookie) { if (likely(doing_pdma)) { void __iomem *stat = (void __iomem *) fdc_status; @@ -254,7 +254,7 @@ static int sun_fd_request_irq(void) once = 1; error = request_irq(FLOPPY_IRQ, sparc_floppy_irq, - IRQF_DISABLED, "floppy", NULL); + 0, "floppy", NULL); return ((error == 0) ? 0 : -1); } @@ -296,21 +296,23 @@ struct sun_pci_dma_op { static struct sun_pci_dma_op sun_pci_dma_current = { -1U, 0, 0, NULL}; static struct sun_pci_dma_op sun_pci_dma_pending = { -1U, 0, 0, NULL}; -extern irqreturn_t floppy_interrupt(int irq, void *dev_id); +irqreturn_t floppy_interrupt(int irq, void *dev_id); -static unsigned char sun_pci_fd_inb(unsigned long port) +static unsigned char sun_pci_fd_inb(unsigned long base, unsigned int reg) { udelay(5); - return inb(port); + return inb(base + reg); } -static void sun_pci_fd_outb(unsigned char val, unsigned long port) +static void sun_pci_fd_outb(unsigned char val, unsigned long base, + unsigned int reg) { udelay(5); - outb(val, port); + outb(val, base + reg); } -static void sun_pci_fd_broken_outb(unsigned char val, unsigned long port) +static void sun_pci_fd_broken_outb(unsigned char val, unsigned long base, + unsigned int reg) { udelay(5); /* @@ -320,16 +322,17 @@ static void sun_pci_fd_broken_outb(unsigned char val, unsigned long port) * this does not hurt correct hardware like the AXmp. * (Eddie, Sep 12 1998). */ - if (port == ((unsigned long)sun_fdc) + 2) { + if (reg == FD_DOR) { if (((val & 0x03) == sun_pci_broken_drive) && (val & 0x20)) { val |= 0x10; } } - outb(val, port); + outb(val, base + reg); } #ifdef PCI_FDC_SWAP_DRIVES -static void sun_pci_fd_lde_broken_outb(unsigned char val, unsigned long port) +static void sun_pci_fd_lde_broken_outb(unsigned char val, unsigned long base, + unsigned int reg) { udelay(5); /* @@ -339,13 +342,13 @@ static void sun_pci_fd_lde_broken_outb(unsigned char val, unsigned long port) * this does not hurt correct hardware like the AXmp. * (Eddie, Sep 12 1998). */ - if (port == ((unsigned long)sun_fdc) + 2) { + if (reg == FD_DOR) { if (((val & 0x03) == sun_pci_broken_drive) && (val & 0x10)) { val &= ~(0x03); val |= 0x21; } } - outb(val, port); + outb(val, base + reg); } #endif /* PCI_FDC_SWAP_DRIVES */ @@ -429,7 +432,8 @@ static int sun_pci_fd_eject(int drive) return -EINVAL; } -void sun_pci_fd_dma_callback(struct ebus_dma_info *p, int event, void *cookie) +static void sun_pci_fd_dma_callback(struct ebus_dma_info *p, int event, + void *cookie) { floppy_interrupt(0, NULL); } @@ -528,9 +532,9 @@ static int sun_pci_fd_test_drive(unsigned long port, int drive) static int __init ebus_fdthree_p(struct device_node *dp) { - if (!strcmp(dp->name, "fdthree")) + if (of_node_name_eq(dp, "fdthree")) return 1; - if (!strcmp(dp->name, "floppy")) { + if (of_node_name_eq(dp, "floppy")) { const char *compat; compat = of_get_property(dp, "compatible", NULL); @@ -555,7 +559,7 @@ static unsigned long __init sun_floppy_init(void) op = NULL; for_each_node_by_name(dp, "SUNW,fdtwo") { - if (strcmp(dp->parent->name, "sbus")) + if (!of_node_name_eq(dp->parent, "sbus")) continue; op = of_find_device_by_node(dp); if (op) @@ -612,7 +616,7 @@ static unsigned long __init sun_floppy_init(void) sun_pci_fd_ebus_dma.callback = sun_pci_fd_dma_callback; sun_pci_fd_ebus_dma.client_cookie = NULL; sun_pci_fd_ebus_dma.irq = FLOPPY_IRQ; - strcpy(sun_pci_fd_ebus_dma.name, "floppy"); + strscpy(sun_pci_fd_ebus_dma.name, "floppy"); if (ebus_dma_register(&sun_pci_fd_ebus_dma)) return 0; @@ -656,7 +660,7 @@ static unsigned long __init sun_floppy_init(void) */ config = 0; for (dp = ebus_dp->child; dp; dp = dp->sibling) { - if (!strcmp(dp->name, "ecpp")) { + if (of_node_name_eq(dp, "ecpp")) { struct platform_device *ecpp_op; ecpp_op = of_find_device_by_node(dp); @@ -699,9 +703,7 @@ static unsigned long __init sun_floppy_init(void) ns87303_modify(config, ASC, ASC_DRV2_SEL, 0); ns87303_modify(config, FCR, 0, FCR_LDE); - config = sun_floppy_types[0]; - sun_floppy_types[0] = sun_floppy_types[1]; - sun_floppy_types[1] = config; + swap(sun_floppy_types[0], sun_floppy_types[1]); if (sun_pci_broken_drive != -1) { sun_pci_broken_drive = 1 - sun_pci_broken_drive; diff --git a/arch/sparc/include/asm/fpumacro.h b/arch/sparc/include/asm/fpumacro.h index cc463fec806f..bc378df97fa8 100644 --- a/arch/sparc/include/asm/fpumacro.h +++ b/arch/sparc/include/asm/fpumacro.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* fpumacro.h: FPU related macros. * * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) diff --git a/arch/sparc/include/asm/ftrace.h b/arch/sparc/include/asm/ftrace.h index b0f18e9893db..f7c9036199c5 100644 --- a/arch/sparc/include/asm/ftrace.h +++ b/arch/sparc/include/asm/ftrace.h @@ -1,18 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_SPARC64_FTRACE #define _ASM_SPARC64_FTRACE #ifdef CONFIG_MCOUNT -#define MCOUNT_ADDR ((long)(_mcount)) +#define MCOUNT_ADDR ((unsigned long)(_mcount)) #define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */ -#ifndef __ASSEMBLY__ -extern void _mcount(void); +#ifndef __ASSEMBLER__ +void _mcount(void); #endif +#endif /* CONFIG_MCOUNT */ + +#if defined(CONFIG_SPARC64) && !defined(CC_USE_FENTRY) +#define HAVE_FUNCTION_GRAPH_FP_TEST #endif #ifdef CONFIG_DYNAMIC_FTRACE -/* reloction of mcount call site is the same as the address */ +/* relocation of mcount call site is the same as the address */ static inline unsigned long ftrace_call_adjust(unsigned long addr) { return addr; @@ -22,4 +27,8 @@ struct dyn_arch_ftrace { }; #endif /* CONFIG_DYNAMIC_FTRACE */ +unsigned long prepare_ftrace_return(unsigned long parent, + unsigned long self_addr, + unsigned long frame_pointer); + #endif /* _ASM_SPARC64_FTRACE */ diff --git a/arch/sparc/include/asm/futex.h b/arch/sparc/include/asm/futex.h index 736335f36713..75a13a20cc64 100644 --- a/arch/sparc/include/asm/futex.h +++ b/arch/sparc/include/asm/futex.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_FUTEX_H #define ___ASM_SPARC_FUTEX_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/futex_64.h b/arch/sparc/include/asm/futex_64.h index 4e899b0dabf7..72de967318d7 100644 --- a/arch/sparc/include/asm/futex_64.h +++ b/arch/sparc/include/asm/futex_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_FUTEX_H #define _SPARC64_FUTEX_H @@ -29,24 +30,14 @@ : "r" (uaddr), "r" (oparg), "i" (-EFAULT) \ : "memory") -static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr) +static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, + u32 __user *uaddr) { - int op = (encoded_op >> 28) & 7; - int cmp = (encoded_op >> 24) & 15; - int oparg = (encoded_op << 8) >> 20; - int cmparg = (encoded_op << 20) >> 20; int oldval = 0, ret, tem; - if (unlikely(!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))) - return -EFAULT; if (unlikely((((unsigned long) uaddr) & 0x3UL))) return -EINVAL; - if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) - oparg = 1 << oparg; - - pagefault_disable(); - switch (op) { case FUTEX_OP_SET: __futex_cas_op("mov\t%4, %1", ret, oldval, uaddr, oparg); @@ -67,19 +58,9 @@ static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr) ret = -ENOSYS; } - pagefault_enable(); + if (!ret) + *oval = oldval; - if (!ret) { - switch (cmp) { - case FUTEX_OP_CMP_EQ: ret = (oldval == cmparg); break; - case FUTEX_OP_CMP_NE: ret = (oldval != cmparg); break; - case FUTEX_OP_CMP_LT: ret = (oldval < cmparg); break; - case FUTEX_OP_CMP_GE: ret = (oldval >= cmparg); break; - case FUTEX_OP_CMP_LE: ret = (oldval <= cmparg); break; - case FUTEX_OP_CMP_GT: ret = (oldval > cmparg); break; - default: ret = -ENOSYS; - } - } return ret; } diff --git a/arch/sparc/include/asm/gpio.h b/arch/sparc/include/asm/gpio.h deleted file mode 100644 index b3799d88ffcf..000000000000 --- a/arch/sparc/include/asm/gpio.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef __LINUX_GPIO_H -#warning Include linux/gpio.h instead of asm/gpio.h -#include <linux/gpio.h> -#endif diff --git a/arch/sparc/include/asm/hardirq.h b/arch/sparc/include/asm/hardirq.h index 44d4e2345148..a185f6647348 100644 --- a/arch/sparc/include/asm/hardirq.h +++ b/arch/sparc/include/asm/hardirq.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_HARDIRQ_H #define ___ASM_SPARC_HARDIRQ_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/hardirq_32.h b/arch/sparc/include/asm/hardirq_32.h index 162007643cdc..9830d8731a09 100644 --- a/arch/sparc/include/asm/hardirq_32.h +++ b/arch/sparc/include/asm/hardirq_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* hardirq.h: 32-bit Sparc hard IRQ support. * * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) @@ -7,7 +8,6 @@ #ifndef __SPARC_HARDIRQ_H #define __SPARC_HARDIRQ_H -#define HARDIRQ_BITS 8 #include <asm-generic/hardirq.h> #endif /* __SPARC_HARDIRQ_H */ diff --git a/arch/sparc/include/asm/hardirq_64.h b/arch/sparc/include/asm/hardirq_64.h index 7c29fd1a87aa..75b92bfe04b5 100644 --- a/arch/sparc/include/asm/hardirq_64.h +++ b/arch/sparc/include/asm/hardirq_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* hardirq.h: 64-bit Sparc hard IRQ support. * * Copyright (C) 1997, 1998, 2005 David S. Miller (davem@davemloft.net) @@ -9,11 +10,10 @@ #include <asm/cpudata.h> #define __ARCH_IRQ_STAT -#define local_softirq_pending() \ - (local_cpu_data().__softirq_pending) -void ack_bad_irq(unsigned int irq); +#define local_softirq_pending_ref \ + __cpu_data.__softirq_pending -#define HARDIRQ_BITS 8 +void ack_bad_irq(unsigned int irq); #endif /* !(__SPARC64_HARDIRQ_H) */ diff --git a/arch/sparc/include/asm/head.h b/arch/sparc/include/asm/head.h index be8f03f3e731..25299b701df5 100644 --- a/arch/sparc/include/asm/head.h +++ b/arch/sparc/include/asm/head.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_HEAD_H #define ___ASM_SPARC_HEAD_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/head_32.h b/arch/sparc/include/asm/head_32.h index 5f1dbe315bc8..d2809c859d0c 100644 --- a/arch/sparc/include/asm/head_32.h +++ b/arch/sparc/include/asm/head_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_HEAD_H #define __SPARC_HEAD_H @@ -43,10 +44,10 @@ nop; #ifdef CONFIG_KGDB -#define KGDB_TRAP(num) \ - b kgdb_trap_low; \ - rd %psr,%l0; \ - nop; \ +#define KGDB_TRAP(num) \ + mov num, %l7; \ + b kgdb_trap_low; \ + rd %psr,%l0; \ nop; #else #define KGDB_TRAP(num) \ diff --git a/arch/sparc/include/asm/head_64.h b/arch/sparc/include/asm/head_64.h index 10e9dabc4c41..69a2062d992c 100644 --- a/arch/sparc/include/asm/head_64.h +++ b/arch/sparc/include/asm/head_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_HEAD_H #define _SPARC64_HEAD_H @@ -15,6 +16,10 @@ #define PTREGS_OFF (STACK_BIAS + STACKFRAME_SZ) +#define RTRAP_PSTATE (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_IE) +#define RTRAP_PSTATE_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV) +#define RTRAP_PSTATE_AG_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_AG) + #define __CHEETAH_ID 0x003e0014 #define __JALAPENO_ID 0x003e0016 #define __SERRANO_ID 0x003e0022 diff --git a/arch/sparc/include/asm/hibernate.h b/arch/sparc/include/asm/hibernate.h index 2ec34f842249..3bb0a96ec4ef 100644 --- a/arch/sparc/include/asm/hibernate.h +++ b/arch/sparc/include/asm/hibernate.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * hibernate.h: Hibernaton support specific for sparc64. * diff --git a/arch/sparc/include/asm/highmem.h b/arch/sparc/include/asm/highmem.h index 4f9e15c757e2..c7b2e208328b 100644 --- a/arch/sparc/include/asm/highmem.h +++ b/arch/sparc/include/asm/highmem.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * highmem.h: virtual kernel memory mappings for high memory * @@ -21,18 +22,16 @@ #ifdef __KERNEL__ #include <linux/interrupt.h> +#include <linux/pgtable.h> #include <asm/vaddrs.h> -#include <asm/kmap_types.h> -#include <asm/pgtable.h> +#include <asm/pgtsrmmu.h> /* declarations for highmem.c */ extern unsigned long highstart_pfn, highend_pfn; -extern pgprot_t kmap_prot; +#define kmap_prot __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE) extern pte_t *pkmap_page_table; -extern void kmap_init(void) __init; - /* * Right now we initialize only a single pte table. It can be extended * easily, subsequent pte tables have to be allocated in one physical @@ -49,30 +48,14 @@ extern void kmap_init(void) __init; #define PKMAP_END (PKMAP_ADDR(LAST_PKMAP)) -extern void *kmap_high(struct page *page); -extern void kunmap_high(struct page *page); - -static inline void *kmap(struct page *page) -{ - BUG_ON(in_interrupt()); - if (!PageHighMem(page)) - return page_address(page); - return kmap_high(page); -} - -static inline void kunmap(struct page *page) -{ - BUG_ON(in_interrupt()); - if (!PageHighMem(page)) - return; - kunmap_high(page); -} - -extern void *kmap_atomic(struct page *page); -extern void __kunmap_atomic(void *kvaddr); - #define flush_cache_kmaps() flush_cache_all() +/* FIXME: Use __flush_*_one(vaddr) instead of flush_*_all() -- Anton */ +#define arch_kmap_local_pre_map(vaddr, pteval) flush_cache_all() +#define arch_kmap_local_pre_unmap(vaddr) flush_cache_all() +#define arch_kmap_local_post_map(vaddr, pteval) flush_tlb_all() +#define arch_kmap_local_post_unmap(vaddr) flush_tlb_all() + #endif /* __KERNEL__ */ #endif /* _ASM_HIGHMEM_H */ diff --git a/arch/sparc/include/asm/hugetlb.h b/arch/sparc/include/asm/hugetlb.h index e4cab465b81f..d3bc16fbcbbd 100644 --- a/arch/sparc/include/asm/hugetlb.h +++ b/arch/sparc/include/asm/hugetlb.h @@ -1,98 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_SPARC64_HUGETLB_H #define _ASM_SPARC64_HUGETLB_H #include <asm/page.h> -#include <asm-generic/hugetlb.h> +#ifdef CONFIG_HUGETLB_PAGE +struct pud_huge_patch_entry { + unsigned int addr; + unsigned int insn; +}; +extern struct pud_huge_patch_entry __pud_huge_patch, __pud_huge_patch_end; +#endif +#define __HAVE_ARCH_HUGE_SET_HUGE_PTE_AT void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned long sz); +void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte); +#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, - pte_t *ptep); - -static inline void hugetlb_prefault_arch_hook(struct mm_struct *mm) -{ -} - -static inline int is_hugepage_only_range(struct mm_struct *mm, - unsigned long addr, - unsigned long len) { - return 0; -} - -/* - * If the arch doesn't supply something else, assume that hugepage - * size aligned regions are ok without further preparation. - */ -static inline int prepare_hugepage_range(struct file *file, - unsigned long addr, unsigned long len) -{ - if (len & ~HPAGE_MASK) - return -EINVAL; - if (addr & ~HPAGE_MASK) - return -EINVAL; - return 0; -} - -static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb, - unsigned long addr, unsigned long end, - unsigned long floor, - unsigned long ceiling) -{ - free_pgd_range(tlb, addr, end, floor, ceiling); -} - -static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) -{ -} - -static inline int huge_pte_none(pte_t pte) -{ - return pte_none(pte); -} + pte_t *ptep, unsigned long sz); -static inline pte_t huge_pte_wrprotect(pte_t pte) +#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH +static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) { - return pte_wrprotect(pte); + return *ptep; } +#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { pte_t old_pte = *ptep; - set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte)); + __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte)); } +#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t pte, int dirty) { int changed = !pte_same(*ptep, pte); if (changed) { - set_huge_pte_at(vma->vm_mm, addr, ptep, pte); + __set_huge_pte_at(vma->vm_mm, addr, ptep, pte); flush_tlb_page(vma, addr); } return changed; } -static inline pte_t huge_ptep_get(pte_t *ptep) -{ - return *ptep; -} - -static inline int arch_prepare_hugepage(struct page *page) -{ - return 0; -} - -static inline void arch_release_hugepage(struct page *page) -{ -} - -static inline void arch_clear_hugepage_flags(struct page *page) -{ -} +#include <asm-generic/hugetlb.h> #endif /* _ASM_SPARC64_HUGETLB_H */ diff --git a/arch/sparc/include/asm/hvtramp.h b/arch/sparc/include/asm/hvtramp.h index b2b9b947b3a4..8cf7a54fa528 100644 --- a/arch/sparc/include/asm/hvtramp.h +++ b/arch/sparc/include/asm/hvtramp.h @@ -1,7 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_HVTRAP_H #define _SPARC64_HVTRAP_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> @@ -16,10 +17,10 @@ struct hvtramp_descr { __u64 fault_info_va; __u64 fault_info_pa; __u64 thread_reg; - struct hvtramp_mapping maps[1]; + struct hvtramp_mapping maps[]; }; -extern void hv_cpu_startup(unsigned long hvdescr_pa); +void hv_cpu_startup(unsigned long hvdescr_pa); #endif diff --git a/arch/sparc/include/asm/hypervisor.h b/arch/sparc/include/asm/hypervisor.h index ca121f0fa3ec..94ac56d43746 100644 --- a/arch/sparc/include/asm/hypervisor.h +++ b/arch/sparc/include/asm/hypervisor.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_HYPERVISOR_H #define _SPARC64_HYPERVISOR_H @@ -75,6 +76,10 @@ #define HV_ETOOMANY 15 /* Too many items specified */ #define HV_ECHANNEL 16 /* Invalid LDC channel */ #define HV_EBUSY 17 /* Resource busy */ +#define HV_EUNAVAILABLE 23 /* Resource or operation not + * currently available, but may + * become available in the future + */ /* mach_exit() * TRAP: HV_FAST_TRAP @@ -97,8 +102,8 @@ */ #define HV_FAST_MACH_EXIT 0x00 -#ifndef __ASSEMBLY__ -extern void sun4v_mach_exit(unsigned long exit_code); +#ifndef __ASSEMBLER__ +void sun4v_mach_exit(unsigned long exit_code); #endif /* Domain services. */ @@ -126,10 +131,10 @@ extern void sun4v_mach_exit(unsigned long exit_code); */ #define HV_FAST_MACH_DESC 0x01 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_mach_desc(unsigned long buffer_pa, - unsigned long buf_len, - unsigned long *real_buf_len); +#ifndef __ASSEMBLER__ +unsigned long sun4v_mach_desc(unsigned long buffer_pa, + unsigned long buf_len, + unsigned long *real_buf_len); #endif /* mach_sir() @@ -147,8 +152,8 @@ extern unsigned long sun4v_mach_desc(unsigned long buffer_pa, */ #define HV_FAST_MACH_SIR 0x02 -#ifndef __ASSEMBLY__ -extern void sun4v_mach_sir(void); +#ifndef __ASSEMBLER__ +void sun4v_mach_sir(void); #endif /* mach_set_watchdog() @@ -203,9 +208,9 @@ extern void sun4v_mach_sir(void); */ #define HV_FAST_MACH_SET_WATCHDOG 0x05 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_mach_set_watchdog(unsigned long timeout, - unsigned long *orig_timeout); +#ifndef __ASSEMBLER__ +unsigned long sun4v_mach_set_watchdog(unsigned long timeout, + unsigned long *orig_timeout); #endif /* CPU services. @@ -249,11 +254,11 @@ extern unsigned long sun4v_mach_set_watchdog(unsigned long timeout, */ #define HV_FAST_CPU_START 0x10 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_cpu_start(unsigned long cpuid, - unsigned long pc, - unsigned long rtba, - unsigned long arg0); +#ifndef __ASSEMBLER__ +unsigned long sun4v_cpu_start(unsigned long cpuid, + unsigned long pc, + unsigned long rtba, + unsigned long arg0); #endif /* cpu_stop() @@ -277,8 +282,8 @@ extern unsigned long sun4v_cpu_start(unsigned long cpuid, */ #define HV_FAST_CPU_STOP 0x11 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_cpu_stop(unsigned long cpuid); +#ifndef __ASSEMBLER__ +unsigned long sun4v_cpu_stop(unsigned long cpuid); #endif /* cpu_yield() @@ -294,8 +299,26 @@ extern unsigned long sun4v_cpu_stop(unsigned long cpuid); */ #define HV_FAST_CPU_YIELD 0x12 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_cpu_yield(void); +#ifndef __ASSEMBLER__ +unsigned long sun4v_cpu_yield(void); +#endif + +/* cpu_poke() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CPU_POKE + * RET0: status + * ERRORS: ENOCPU cpuid refers to a CPU that does not exist + * EINVAL cpuid is current CPU + * + * Poke CPU cpuid. If the target CPU is currently suspended having + * invoked the cpu-yield service, that vCPU will be resumed. + * Poke interrupts may only be sent to valid, non-local CPUs. + * It is not legal to poke the current vCPU. + */ +#define HV_FAST_CPU_POKE 0x13 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_cpu_poke(unsigned long cpuid); #endif /* cpu_qconf() @@ -340,10 +363,10 @@ extern unsigned long sun4v_cpu_yield(void); #define HV_CPU_QUEUE_RES_ERROR 0x3e #define HV_CPU_QUEUE_NONRES_ERROR 0x3f -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_cpu_qconf(unsigned long type, - unsigned long queue_paddr, - unsigned long num_queue_entries); +#ifndef __ASSEMBLER__ +unsigned long sun4v_cpu_qconf(unsigned long type, + unsigned long queue_paddr, + unsigned long num_queue_entries); #endif /* cpu_qinfo() @@ -393,8 +416,10 @@ extern unsigned long sun4v_cpu_qconf(unsigned long type, */ #define HV_FAST_CPU_MONDO_SEND 0x42 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_cpu_mondo_send(unsigned long cpu_count, unsigned long cpu_list_pa, unsigned long mondo_block_pa); +#ifndef __ASSEMBLER__ +unsigned long sun4v_cpu_mondo_send(unsigned long cpu_count, + unsigned long cpu_list_pa, + unsigned long mondo_block_pa); #endif /* cpu_myid() @@ -405,7 +430,7 @@ extern unsigned long sun4v_cpu_mondo_send(unsigned long cpu_count, unsigned long * ERRORS: No errors defined. * * Return the hypervisor ID handle for the current CPU. Use by a - * virtual CPU to discover it's own identity. + * virtual CPU to discover its own identity. */ #define HV_FAST_CPU_MYID 0x16 @@ -424,8 +449,8 @@ extern unsigned long sun4v_cpu_mondo_send(unsigned long cpu_count, unsigned long #define HV_CPU_STATE_RUNNING 0x02 #define HV_CPU_STATE_ERROR 0x03 -#ifndef __ASSEMBLY__ -extern long sun4v_cpu_state(unsigned long cpuid); +#ifndef __ASSEMBLER__ +long sun4v_cpu_state(unsigned long cpuid); #endif /* cpu_set_rtba() @@ -460,7 +485,7 @@ extern long sun4v_cpu_state(unsigned long cpuid); * * Layout of a TSB description for mmu_tsb_ctx{,non}0() calls. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct hv_tsb_descr { unsigned short pgsz_idx; unsigned short assoc; @@ -511,7 +536,7 @@ struct hv_tsb_descr { * The fault status block is a multiple of 64-bytes and must be aligned * on a 64-byte boundary. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct hv_fault_status { unsigned long i_fault_type; unsigned long i_fault_addr; @@ -545,6 +570,8 @@ struct hv_fault_status { #define HV_FAULT_TYPE_RESV1 13 #define HV_FAULT_TYPE_UNALIGNED 14 #define HV_FAULT_TYPE_INV_PGSZ 15 +#define HV_FAULT_TYPE_MCD 17 +#define HV_FAULT_TYPE_MCD_DIS 18 /* Values 16 --> -2 are reserved. */ #define HV_FAULT_TYPE_MULTIPLE -1 @@ -624,9 +651,9 @@ struct hv_fault_status { */ #define HV_FAST_MMU_TSB_CTX0 0x20 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_mmu_tsb_ctx0(unsigned long num_descriptions, - unsigned long tsb_desc_ra); +#ifndef __ASSEMBLER__ +unsigned long sun4v_mmu_tsb_ctx0(unsigned long num_descriptions, + unsigned long tsb_desc_ra); #endif /* mmu_tsb_ctxnon0() @@ -709,8 +736,8 @@ extern unsigned long sun4v_mmu_tsb_ctx0(unsigned long num_descriptions, */ #define HV_FAST_MMU_DEMAP_ALL 0x24 -#ifndef __ASSEMBLY__ -extern void sun4v_mmu_demap_all(void); +#ifndef __ASSEMBLER__ +void sun4v_mmu_demap_all(void); #endif /* mmu_map_perm_addr() @@ -739,11 +766,11 @@ extern void sun4v_mmu_demap_all(void); */ #define HV_FAST_MMU_MAP_PERM_ADDR 0x25 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_mmu_map_perm_addr(unsigned long vaddr, - unsigned long set_to_zero, - unsigned long tte, - unsigned long flags); +#ifndef __ASSEMBLER__ +unsigned long sun4v_mmu_map_perm_addr(unsigned long vaddr, + unsigned long set_to_zero, + unsigned long tte, + unsigned long flags); #endif /* mmu_fault_area_conf() @@ -920,6 +947,139 @@ extern unsigned long sun4v_mmu_map_perm_addr(unsigned long vaddr, */ #define HV_FAST_MEM_SYNC 0x32 +/* Coprocessor services + * + * M7 and later processors provide an on-chip coprocessor which + * accelerates database operations, and is known internally as + * DAX. + */ + +/* ccb_submit() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_CCB_SUBMIT + * ARG0: address of CCB array + * ARG1: size (in bytes) of CCB array being submitted + * ARG2: flags + * ARG3: reserved + * RET0: status (success or error code) + * RET1: size (in bytes) of CCB array that was accepted (might be less + * than arg1) + * RET2: status data + * if status == ENOMAP or ENOACCESS, identifies the VA in question + * if status == EUNAVAILBLE, unavailable code + * RET3: reserved + * + * ERRORS: EOK successful submission (check size) + * EWOULDBLOCK could not finish submissions, try again + * EBADALIGN array not 64B aligned or size not 64B multiple + * ENORADDR invalid RA for array or in CCB + * ENOMAP could not translate address (see status data) + * EINVAL invalid ccb or arguments + * ETOOMANY too many ccbs with all-or-nothing flag + * ENOACCESS guest has no access to submit ccbs or address + * in CCB does not have correct permissions (check + * status data) + * EUNAVAILABLE ccb operation could not be performed at this + * time (check status data) + * Status data codes: + * 0 - exact CCB could not be executed + * 1 - CCB opcode cannot be executed + * 2 - CCB version cannot be executed + * 3 - vcpu cannot execute CCBs + * 4 - no CCBs can be executed + */ + +#define HV_CCB_SUBMIT 0x34 +#ifndef __ASSEMBLER__ +unsigned long sun4v_ccb_submit(unsigned long ccb_buf, + unsigned long len, + unsigned long flags, + unsigned long reserved, + void *submitted_len, + void *status_data); +#endif + +/* flags (ARG2) */ +#define HV_CCB_QUERY_CMD BIT(1) +#define HV_CCB_ARG0_TYPE_REAL 0UL +#define HV_CCB_ARG0_TYPE_PRIMARY BIT(4) +#define HV_CCB_ARG0_TYPE_SECONDARY BIT(5) +#define HV_CCB_ARG0_TYPE_NUCLEUS GENMASK(5, 4) +#define HV_CCB_ARG0_PRIVILEGED BIT(6) +#define HV_CCB_ALL_OR_NOTHING BIT(7) +#define HV_CCB_QUEUE_INFO BIT(8) +#define HV_CCB_VA_REJECT 0UL +#define HV_CCB_VA_SECONDARY BIT(13) +#define HV_CCB_VA_NUCLEUS GENMASK(13, 12) +#define HV_CCB_VA_PRIVILEGED BIT(14) +#define HV_CCB_VA_READ_ADI_DISABLE BIT(15) /* DAX2 only */ + +/* ccb_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_CCB_INFO + * ARG0: real address of CCB completion area + * RET0: status (success or error code) + * RET1: info array + * - RET1[0]: CCB state + * - RET1[1]: dax unit + * - RET1[2]: queue number + * - RET1[3]: queue position + * + * ERRORS: EOK operation successful + * EBADALIGN address not 64B aligned + * ENORADDR RA in address not valid + * EINVAL CA not valid + * EWOULDBLOCK info not available for this CCB currently, try + * again + * ENOACCESS guest cannot use dax + */ + +#define HV_CCB_INFO 0x35 +#ifndef __ASSEMBLER__ +unsigned long sun4v_ccb_info(unsigned long ca, + void *info_arr); +#endif + +/* info array byte offsets (RET1) */ +#define CCB_INFO_OFFSET_CCB_STATE 0 +#define CCB_INFO_OFFSET_DAX_UNIT 2 +#define CCB_INFO_OFFSET_QUEUE_NUM 4 +#define CCB_INFO_OFFSET_QUEUE_POS 6 + +/* CCB state (RET1[0]) */ +#define HV_CCB_STATE_COMPLETED 0 +#define HV_CCB_STATE_ENQUEUED 1 +#define HV_CCB_STATE_INPROGRESS 2 +#define HV_CCB_STATE_NOTFOUND 3 + +/* ccb_kill() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_CCB_KILL + * ARG0: real address of CCB completion area + * RET0: status (success or error code) + * RET1: CCB kill status + * + * ERRORS: EOK operation successful + * EBADALIGN address not 64B aligned + * ENORADDR RA in address not valid + * EINVAL CA not valid + * EWOULDBLOCK kill not available for this CCB currently, try + * again + * ENOACCESS guest cannot use dax + */ + +#define HV_CCB_KILL 0x36 +#ifndef __ASSEMBLER__ +unsigned long sun4v_ccb_kill(unsigned long ca, + void *kill_status); +#endif + +/* CCB kill status (RET1) */ +#define HV_CCB_KILL_COMPLETED 0 +#define HV_CCB_KILL_DEQUEUED 1 +#define HV_CCB_KILL_KILLED 2 +#define HV_CCB_KILL_NOTFOUND 3 + /* Time of day services. * * The hypervisor maintains the time of day on a per-domain basis. @@ -944,8 +1104,8 @@ extern unsigned long sun4v_mmu_map_perm_addr(unsigned long vaddr, */ #define HV_FAST_TOD_GET 0x50 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_tod_get(unsigned long *time); +#ifndef __ASSEMBLER__ +unsigned long sun4v_tod_get(unsigned long *time); #endif /* tod_set() @@ -961,8 +1121,8 @@ extern unsigned long sun4v_tod_get(unsigned long *time); */ #define HV_FAST_TOD_SET 0x51 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_tod_set(unsigned long time); +#ifndef __ASSEMBLER__ +unsigned long sun4v_tod_set(unsigned long time); #endif /* Console services */ @@ -1037,15 +1197,15 @@ extern unsigned long sun4v_tod_set(unsigned long time); */ #define HV_FAST_CONS_WRITE 0x63 -#ifndef __ASSEMBLY__ -extern long sun4v_con_getchar(long *status); -extern long sun4v_con_putchar(long c); -extern long sun4v_con_read(unsigned long buffer, - unsigned long size, - unsigned long *bytes_read); -extern unsigned long sun4v_con_write(unsigned long buffer, - unsigned long size, - unsigned long *bytes_written); +#ifndef __ASSEMBLER__ +long sun4v_con_getchar(long *status); +long sun4v_con_putchar(long c); +long sun4v_con_read(unsigned long buffer, + unsigned long size, + unsigned long *bytes_read); +unsigned long sun4v_con_write(unsigned long buffer, + unsigned long size, + unsigned long *bytes_written); #endif /* mach_set_soft_state() @@ -1061,7 +1221,7 @@ extern unsigned long sun4v_con_write(unsigned long buffer, * EBADALIGNED software state description is not correctly * aligned * - * This allows the guest to report it's soft state to the hypervisor. There + * This allows the guest to report its soft state to the hypervisor. There * are two primary components to this state. The first part states whether * the guest software is running or not. The second containts optional * details specific to the software. @@ -1079,9 +1239,9 @@ extern unsigned long sun4v_con_write(unsigned long buffer, #define HV_SOFT_STATE_NORMAL 0x01 #define HV_SOFT_STATE_TRANSITION 0x02 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_mach_set_soft_state(unsigned long soft_state, - unsigned long msg_string_ra); +#ifndef __ASSEMBLER__ +unsigned long sun4v_mach_set_soft_state(unsigned long soft_state, + unsigned long msg_string_ra); #endif /* mach_get_soft_state() @@ -1158,21 +1318,21 @@ extern unsigned long sun4v_mach_set_soft_state(unsigned long soft_state, */ #define HV_FAST_SVC_CLRSTATUS 0x84 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_svc_send(unsigned long svc_id, - unsigned long buffer, - unsigned long buffer_size, - unsigned long *sent_bytes); -extern unsigned long sun4v_svc_recv(unsigned long svc_id, - unsigned long buffer, - unsigned long buffer_size, - unsigned long *recv_bytes); -extern unsigned long sun4v_svc_getstatus(unsigned long svc_id, - unsigned long *status_bits); -extern unsigned long sun4v_svc_setstatus(unsigned long svc_id, - unsigned long status_bits); -extern unsigned long sun4v_svc_clrstatus(unsigned long svc_id, - unsigned long status_bits); +#ifndef __ASSEMBLER__ +unsigned long sun4v_svc_send(unsigned long svc_id, + unsigned long buffer, + unsigned long buffer_size, + unsigned long *sent_bytes); +unsigned long sun4v_svc_recv(unsigned long svc_id, + unsigned long buffer, + unsigned long buffer_size, + unsigned long *recv_bytes); +unsigned long sun4v_svc_getstatus(unsigned long svc_id, + unsigned long *status_bits); +unsigned long sun4v_svc_setstatus(unsigned long svc_id, + unsigned long status_bits); +unsigned long sun4v_svc_clrstatus(unsigned long svc_id, + unsigned long status_bits); #endif /* Trap trace services. @@ -1188,7 +1348,7 @@ extern unsigned long sun4v_svc_clrstatus(unsigned long svc_id, * start (offset 0) of the trap trace buffer, and is described as * follows: */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct hv_trap_trace_control { unsigned long head_offset; unsigned long tail_offset; @@ -1207,7 +1367,7 @@ struct hv_trap_trace_control { * * Each trap trace buffer entry is laid out as follows: */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct hv_trap_trace_entry { unsigned char type; /* Hypervisor or guest entry? */ unsigned char hpstate; /* Hyper-privileged state */ @@ -1342,7 +1502,7 @@ struct hv_trap_trace_entry { * configuration error of some sort. * * The dump services provide an opaque buffer into which the - * hypervisor can place it's internal state in order to assist in + * hypervisor can place its internal state in order to assist in * debugging such situations. The contents are opaque and extremely * platform and hypervisor implementation specific. The guest, during * a core dump, requests that the hypervisor update any information in @@ -1457,9 +1617,9 @@ struct hv_trap_trace_entry { */ #define HV_FAST_INTR_DEVINO2SYSINO 0xa0 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_devino_to_sysino(unsigned long devhandle, - unsigned long devino); +#ifndef __ASSEMBLER__ +unsigned long sun4v_devino_to_sysino(unsigned long devhandle, + unsigned long devino); #endif /* intr_getenabled() @@ -1475,8 +1635,8 @@ extern unsigned long sun4v_devino_to_sysino(unsigned long devhandle, */ #define HV_FAST_INTR_GETENABLED 0xa1 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_intr_getenabled(unsigned long sysino); +#ifndef __ASSEMBLER__ +unsigned long sun4v_intr_getenabled(unsigned long sysino); #endif /* intr_setenabled() @@ -1491,8 +1651,9 @@ extern unsigned long sun4v_intr_getenabled(unsigned long sysino); */ #define HV_FAST_INTR_SETENABLED 0xa2 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_intr_setenabled(unsigned long sysino, unsigned long intr_enabled); +#ifndef __ASSEMBLER__ +unsigned long sun4v_intr_setenabled(unsigned long sysino, + unsigned long intr_enabled); #endif /* intr_getstate() @@ -1507,8 +1668,8 @@ extern unsigned long sun4v_intr_setenabled(unsigned long sysino, unsigned long i */ #define HV_FAST_INTR_GETSTATE 0xa3 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_intr_getstate(unsigned long sysino); +#ifndef __ASSEMBLER__ +unsigned long sun4v_intr_getstate(unsigned long sysino); #endif /* intr_setstate() @@ -1527,8 +1688,8 @@ extern unsigned long sun4v_intr_getstate(unsigned long sysino); */ #define HV_FAST_INTR_SETSTATE 0xa4 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_intr_setstate(unsigned long sysino, unsigned long intr_state); +#ifndef __ASSEMBLER__ +unsigned long sun4v_intr_setstate(unsigned long sysino, unsigned long intr_state); #endif /* intr_gettarget() @@ -1545,8 +1706,8 @@ extern unsigned long sun4v_intr_setstate(unsigned long sysino, unsigned long int */ #define HV_FAST_INTR_GETTARGET 0xa5 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_intr_gettarget(unsigned long sysino); +#ifndef __ASSEMBLER__ +unsigned long sun4v_intr_gettarget(unsigned long sysino); #endif /* intr_settarget() @@ -1562,8 +1723,8 @@ extern unsigned long sun4v_intr_gettarget(unsigned long sysino); */ #define HV_FAST_INTR_SETTARGET 0xa6 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_intr_settarget(unsigned long sysino, unsigned long cpuid); +#ifndef __ASSEMBLER__ +unsigned long sun4v_intr_settarget(unsigned long sysino, unsigned long cpuid); #endif /* vintr_get_cookie() @@ -1646,31 +1807,31 @@ extern unsigned long sun4v_intr_settarget(unsigned long sysino, unsigned long cp */ #define HV_FAST_VINTR_SET_TARGET 0xae -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_vintr_get_cookie(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long *cookie); -extern unsigned long sun4v_vintr_set_cookie(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long cookie); -extern unsigned long sun4v_vintr_get_valid(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long *valid); -extern unsigned long sun4v_vintr_set_valid(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long valid); -extern unsigned long sun4v_vintr_get_state(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long *state); -extern unsigned long sun4v_vintr_set_state(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long state); -extern unsigned long sun4v_vintr_get_target(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long *cpuid); -extern unsigned long sun4v_vintr_set_target(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long cpuid); +#ifndef __ASSEMBLER__ +unsigned long sun4v_vintr_get_cookie(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long *cookie); +unsigned long sun4v_vintr_set_cookie(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long cookie); +unsigned long sun4v_vintr_get_valid(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long *valid); +unsigned long sun4v_vintr_set_valid(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long valid); +unsigned long sun4v_vintr_get_state(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long *state); +unsigned long sun4v_vintr_set_state(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long state); +unsigned long sun4v_vintr_get_target(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long *cpuid); +unsigned long sun4v_vintr_set_target(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long cpuid); #endif /* PCI IO services. @@ -1741,6 +1902,7 @@ extern unsigned long sun4v_vintr_set_target(unsigned long dev_handle, #define HV_PCI_MAP_ATTR_READ 0x01 #define HV_PCI_MAP_ATTR_WRITE 0x02 +#define HV_PCI_MAP_ATTR_RELAXED_ORDER 0x04 #define HV_PCI_DEVICE_BUILD(b,d,f) \ ((((b) & 0xff) << 16) | \ @@ -2331,6 +2493,348 @@ extern unsigned long sun4v_vintr_set_target(unsigned long dev_handle, */ #define HV_FAST_PCI_MSG_SETVALID 0xd3 +/* PCI IOMMU v2 definitions and services + * + * While the PCI IO definitions above is valid IOMMU v2 adds new PCI IO + * definitions and services. + * + * CTE Clump Table Entry. First level table entry in the ATU. + * + * pci_device_list + * A 32-bit aligned list of pci_devices. + * + * pci_device_listp + * real address of a pci_device_list. 32-bit aligned. + * + * iotte IOMMU translation table entry. + * + * iotte_attributes + * IO Attributes for IOMMU v2 mappings. In addition to + * read, write IOMMU v2 supports relax ordering + * + * io_page_list A 64-bit aligned list of real addresses. Each real + * address in an io_page_list must be properly aligned + * to the pagesize of the given IOTSB. + * + * io_page_list_p Real address of an io_page_list, 64-bit aligned. + * + * IOTSB IO Translation Storage Buffer. An aligned table of + * IOTTEs. Each IOTSB has a pagesize, table size, and + * virtual address associated with it that must match + * a pagesize and table size supported by the un-derlying + * hardware implementation. The alignment requirements + * for an IOTSB depend on the pagesize used for that IOTSB. + * Each IOTTE in an IOTSB maps one pagesize-sized page. + * The size of the IOTSB dictates how large of a virtual + * address space the IOTSB is capable of mapping. + * + * iotsb_handle An opaque identifier for an IOTSB. A devhandle plus + * iotsb_handle represents a binding of an IOTSB to a + * PCI root complex. + * + * iotsb_index Zero-based IOTTE number within an IOTSB. + */ + +/* The index_count argument consists of two fields: + * bits 63:48 #iottes and bits 47:0 iotsb_index + */ +#define HV_PCI_IOTSB_INDEX_COUNT(__iottes, __iotsb_index) \ + (((u64)(__iottes) << 48UL) | ((u64)(__iotsb_index))) + +/* pci_iotsb_conf() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_CONF + * ARG0: devhandle + * ARG1: r_addr + * ARG2: size + * ARG3: pagesize + * ARG4: iova + * RET0: status + * RET1: iotsb_handle + * ERRORS: EINVAL Invalid devhandle, size, iova, or pagesize + * EBADALIGN r_addr is not properly aligned + * ENORADDR r_addr is not a valid real address + * ETOOMANY No further IOTSBs may be configured + * EBUSY Duplicate devhandle, raddir, iova combination + * + * Create an IOTSB suitable for the PCI root complex identified by devhandle, + * for the DMA virtual address defined by the argument iova. + * + * r_addr is the properly aligned base address of the IOTSB and size is the + * IOTSB (table) size in bytes.The IOTSB is required to be zeroed prior to + * being configured. If it contains any values other than zeros then the + * behavior is undefined. + * + * pagesize is the size of each page in the IOTSB. Note that the combination of + * size (table size) and pagesize must be valid. + * + * virt is the DMA virtual address this IOTSB will map. + * + * If successful, the opaque 64-bit handle iotsb_handle is returned in ret1. + * Once configured, privileged access to the IOTSB memory is prohibited and + * creates undefined behavior. The only permitted access is indirect via these + * services. + */ +#define HV_FAST_PCI_IOTSB_CONF 0x190 + +/* pci_iotsb_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_INFO + * ARG0: devhandle + * ARG1: iotsb_handle + * RET0: status + * RET1: r_addr + * RET2: size + * RET3: pagesize + * RET4: iova + * RET5: #bound + * ERRORS: EINVAL Invalid devhandle or iotsb_handle + * + * This service returns configuration information about an IOTSB previously + * created with pci_iotsb_conf. + * + * iotsb_handle value 0 may be used with this service to inquire about the + * legacy IOTSB that may or may not exist. If the service succeeds, the return + * values describe the legacy IOTSB and I/O virtual addresses mapped by that + * table. However, the table base address r_addr may contain the value -1 which + * indicates a memory range that cannot be accessed or be reclaimed. + * + * The return value #bound contains the number of PCI devices that iotsb_handle + * is currently bound to. + */ +#define HV_FAST_PCI_IOTSB_INFO 0x191 + +/* pci_iotsb_unconf() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_UNCONF + * ARG0: devhandle + * ARG1: iotsb_handle + * RET0: status + * ERRORS: EINVAL Invalid devhandle or iotsb_handle + * EBUSY The IOTSB is bound and may not be unconfigured + * + * This service unconfigures the IOTSB identified by the devhandle and + * iotsb_handle arguments, previously created with pci_iotsb_conf. + * The IOTSB must not be currently bound to any device or the service will fail + * + * If the call succeeds, iotsb_handle is no longer valid. + */ +#define HV_FAST_PCI_IOTSB_UNCONF 0x192 + +/* pci_iotsb_bind() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_BIND + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: pci_device + * RET0: status + * ERRORS: EINVAL Invalid devhandle, iotsb_handle, or pci_device + * EBUSY A PCI function is already bound to an IOTSB at the same + * address range as specified by devhandle, iotsb_handle. + * + * This service binds the PCI function specified by the argument pci_device to + * the IOTSB specified by the arguments devhandle and iotsb_handle. + * + * The PCI device function is bound to the specified IOTSB with the IOVA range + * specified when the IOTSB was configured via pci_iotsb_conf. If the function + * is already bound then it is unbound first. + */ +#define HV_FAST_PCI_IOTSB_BIND 0x193 + +/* pci_iotsb_unbind() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_UNBIND + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: pci_device + * RET0: status + * ERRORS: EINVAL Invalid devhandle, iotsb_handle, or pci_device + * ENOMAP The PCI function was not bound to the specified IOTSB + * + * This service unbinds the PCI device specified by the argument pci_device + * from the IOTSB identified * by the arguments devhandle and iotsb_handle. + * + * If the PCI device is not bound to the specified IOTSB then this service will + * fail with status ENOMAP + */ +#define HV_FAST_PCI_IOTSB_UNBIND 0x194 + +/* pci_iotsb_get_binding() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_GET_BINDING + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: iova + * RET0: status + * RET1: iotsb_handle + * ERRORS: EINVAL Invalid devhandle, pci_device, or iova + * ENOMAP The PCI function is not bound to an IOTSB at iova + * + * This service returns the IOTSB binding, iotsb_handle, for a given pci_device + * and DMA virtual address, iova. + * + * iova must be the base address of a DMA virtual address range as defined by + * the iommu-address-ranges property in the root complex device node defined + * by the argument devhandle. + */ +#define HV_FAST_PCI_IOTSB_GET_BINDING 0x195 + +/* pci_iotsb_map() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_MAP + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: index_count + * ARG3: iotte_attributes + * ARG4: io_page_list_p + * RET0: status + * RET1: #mapped + * ERRORS: EINVAL Invalid devhandle, iotsb_handle, #iottes, + * iotsb_index or iotte_attributes + * EBADALIGN Improperly aligned io_page_list_p or I/O page + * address in the I/O page list. + * ENORADDR Invalid io_page_list_p or I/O page address in + * the I/O page list. + * + * This service creates and flushes mappings in the IOTSB defined by the + * arguments devhandle, iotsb. + * + * The index_count argument consists of two fields. Bits 63:48 contain #iotte + * and bits 47:0 contain iotsb_index + * + * The first mapping is created in the IOTSB index specified by iotsb_index. + * Subsequent mappings are created at iotsb_index+1 and so on. + * + * The attributes of each mapping are defined by the argument iotte_attributes. + * + * The io_page_list_p specifies the real address of the 64-bit-aligned list of + * #iottes I/O page addresses. Each page address must be a properly aligned + * real address of a page to be mapped in the IOTSB. The first entry in the I/O + * page list contains the real address of the first page, the 2nd entry for the + * 2nd page, and so on. + * + * #iottes must be greater than zero. + * + * The return value #mapped is the actual number of mappings created, which may + * be less than or equal to the argument #iottes. If the function returns + * successfully with a #mapped value less than the requested #iottes then the + * caller should continue to invoke the service with updated iotsb_index, + * #iottes, and io_page_list_p arguments until all pages are mapped. + * + * This service must not be used to demap a mapping. In other words, all + * mappings must be valid and have one or both of the RW attribute bits set. + * + * Note: + * It is implementation-defined whether I/O page real address validity checking + * is done at time mappings are established or deferred until they are + * accessed. + */ +#define HV_FAST_PCI_IOTSB_MAP 0x196 + +/* pci_iotsb_map_one() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_MAP_ONE + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: iotsb_index + * ARG3: iotte_attributes + * ARG4: r_addr + * RET0: status + * ERRORS: EINVAL Invalid devhandle,iotsb_handle, iotsb_index + * or iotte_attributes + * EBADALIGN Improperly aligned r_addr + * ENORADDR Invalid r_addr + * + * This service creates and flushes a single mapping in the IOTSB defined by the + * arguments devhandle, iotsb. + * + * The mapping for the page at r_addr is created at the IOTSB index specified by + * iotsb_index with the attributes iotte_attributes. + * + * This service must not be used to demap a mapping. In other words, the mapping + * must be valid and have one or both of the RW attribute bits set. + * + * Note: + * It is implementation-defined whether I/O page real address validity checking + * is done at time mappings are established or deferred until they are + * accessed. + */ +#define HV_FAST_PCI_IOTSB_MAP_ONE 0x197 + +/* pci_iotsb_demap() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_DEMAP + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: iotsb_index + * ARG3: #iottes + * RET0: status + * RET1: #unmapped + * ERRORS: EINVAL Invalid devhandle, iotsb_handle, iotsb_index or #iottes + * + * This service unmaps and flushes up to #iottes mappings starting at index + * iotsb_index from the IOTSB defined by the arguments devhandle, iotsb. + * + * #iottes must be greater than zero. + * + * The actual number of IOTTEs unmapped is returned in #unmapped and may be less + * than or equal to the requested number of IOTTEs, #iottes. + * + * If #unmapped is less than #iottes, the caller should continue to invoke this + * service with updated iotsb_index and #iottes arguments until all pages are + * demapped. + */ +#define HV_FAST_PCI_IOTSB_DEMAP 0x198 + +/* pci_iotsb_getmap() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_GETMAP + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: iotsb_index + * RET0: status + * RET1: r_addr + * RET2: iotte_attributes + * ERRORS: EINVAL Invalid devhandle, iotsb_handle, or iotsb_index + * ENOMAP No mapping was found + * + * This service returns the mapping specified by index iotsb_index from the + * IOTSB defined by the arguments devhandle, iotsb. + * + * Upon success, the real address of the mapping shall be returned in + * r_addr and thethe IOTTE mapping attributes shall be returned in + * iotte_attributes. + * + * The return value iotte_attributes may not include optional features used in + * the call to create the mapping. + */ +#define HV_FAST_PCI_IOTSB_GETMAP 0x199 + +/* pci_iotsb_sync_mappings() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_SYNC_MAPPINGS + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: iotsb_index + * ARG3: #iottes + * RET0: status + * RET1: #synced + * ERROS: EINVAL Invalid devhandle, iotsb_handle, iotsb_index, or #iottes + * + * This service synchronizes #iottes mappings starting at index iotsb_index in + * the IOTSB defined by the arguments devhandle, iotsb. + * + * #iottes must be greater than zero. + * + * The actual number of IOTTEs synchronized is returned in #synced, which may + * be less than or equal to the requested number, #iottes. + * + * Upon a successful return, #synced is less than #iottes, the caller should + * continue to invoke this service with updated iotsb_index and #iottes + * arguments until all pages are synchronized. + */ +#define HV_FAST_PCI_IOTSB_SYNC_MAPPINGS 0x19a + /* Logical Domain Channel services. */ #define LDC_CHANNEL_DOWN 0 @@ -2543,7 +3047,7 @@ extern unsigned long sun4v_vintr_set_target(unsigned long dev_handle, #define LDC_MTE_SZ64K 0x0000000000000001 /* 64K page */ #define LDC_MTE_SZ8K 0x0000000000000000 /* 8K page */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct ldc_mtable_entry { unsigned long mte; unsigned long cookie; @@ -2626,51 +3130,51 @@ struct ldc_mtable_entry { */ #define HV_FAST_LDC_REVOKE 0xef -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_ldc_tx_qconf(unsigned long channel, - unsigned long ra, - unsigned long num_entries); -extern unsigned long sun4v_ldc_tx_qinfo(unsigned long channel, - unsigned long *ra, - unsigned long *num_entries); -extern unsigned long sun4v_ldc_tx_get_state(unsigned long channel, - unsigned long *head_off, - unsigned long *tail_off, - unsigned long *chan_state); -extern unsigned long sun4v_ldc_tx_set_qtail(unsigned long channel, - unsigned long tail_off); -extern unsigned long sun4v_ldc_rx_qconf(unsigned long channel, - unsigned long ra, - unsigned long num_entries); -extern unsigned long sun4v_ldc_rx_qinfo(unsigned long channel, - unsigned long *ra, - unsigned long *num_entries); -extern unsigned long sun4v_ldc_rx_get_state(unsigned long channel, - unsigned long *head_off, - unsigned long *tail_off, - unsigned long *chan_state); -extern unsigned long sun4v_ldc_rx_set_qhead(unsigned long channel, - unsigned long head_off); -extern unsigned long sun4v_ldc_set_map_table(unsigned long channel, - unsigned long ra, - unsigned long num_entries); -extern unsigned long sun4v_ldc_get_map_table(unsigned long channel, - unsigned long *ra, - unsigned long *num_entries); -extern unsigned long sun4v_ldc_copy(unsigned long channel, - unsigned long dir_code, - unsigned long tgt_raddr, - unsigned long lcl_raddr, - unsigned long len, - unsigned long *actual_len); -extern unsigned long sun4v_ldc_mapin(unsigned long channel, - unsigned long cookie, - unsigned long *ra, - unsigned long *perm); -extern unsigned long sun4v_ldc_unmap(unsigned long ra); -extern unsigned long sun4v_ldc_revoke(unsigned long channel, - unsigned long cookie, - unsigned long mte_cookie); +#ifndef __ASSEMBLER__ +unsigned long sun4v_ldc_tx_qconf(unsigned long channel, + unsigned long ra, + unsigned long num_entries); +unsigned long sun4v_ldc_tx_qinfo(unsigned long channel, + unsigned long *ra, + unsigned long *num_entries); +unsigned long sun4v_ldc_tx_get_state(unsigned long channel, + unsigned long *head_off, + unsigned long *tail_off, + unsigned long *chan_state); +unsigned long sun4v_ldc_tx_set_qtail(unsigned long channel, + unsigned long tail_off); +unsigned long sun4v_ldc_rx_qconf(unsigned long channel, + unsigned long ra, + unsigned long num_entries); +unsigned long sun4v_ldc_rx_qinfo(unsigned long channel, + unsigned long *ra, + unsigned long *num_entries); +unsigned long sun4v_ldc_rx_get_state(unsigned long channel, + unsigned long *head_off, + unsigned long *tail_off, + unsigned long *chan_state); +unsigned long sun4v_ldc_rx_set_qhead(unsigned long channel, + unsigned long head_off); +unsigned long sun4v_ldc_set_map_table(unsigned long channel, + unsigned long ra, + unsigned long num_entries); +unsigned long sun4v_ldc_get_map_table(unsigned long channel, + unsigned long *ra, + unsigned long *num_entries); +unsigned long sun4v_ldc_copy(unsigned long channel, + unsigned long dir_code, + unsigned long tgt_raddr, + unsigned long lcl_raddr, + unsigned long len, + unsigned long *actual_len); +unsigned long sun4v_ldc_mapin(unsigned long channel, + unsigned long cookie, + unsigned long *ra, + unsigned long *perm); +unsigned long sun4v_ldc_unmap(unsigned long ra); +unsigned long sun4v_ldc_revoke(unsigned long channel, + unsigned long cookie, + unsigned long mte_cookie); #endif /* Performance counter services. */ @@ -2726,15 +3230,15 @@ extern unsigned long sun4v_ldc_revoke(unsigned long channel, #define HV_FAST_N2_GET_PERFREG 0x104 #define HV_FAST_N2_SET_PERFREG 0x105 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_niagara_getperf(unsigned long reg, - unsigned long *val); -extern unsigned long sun4v_niagara_setperf(unsigned long reg, - unsigned long val); -extern unsigned long sun4v_niagara2_getperf(unsigned long reg, - unsigned long *val); -extern unsigned long sun4v_niagara2_setperf(unsigned long reg, - unsigned long val); +#ifndef __ASSEMBLER__ +unsigned long sun4v_niagara_getperf(unsigned long reg, + unsigned long *val); +unsigned long sun4v_niagara_setperf(unsigned long reg, + unsigned long val); +unsigned long sun4v_niagara2_getperf(unsigned long reg, + unsigned long *val); +unsigned long sun4v_niagara2_setperf(unsigned long reg, + unsigned long val); #endif /* MMU statistics services. @@ -2743,7 +3247,7 @@ extern unsigned long sun4v_niagara2_setperf(unsigned long reg, * a buffer where these statistics can be collected. It is continually * updated once configured. The layout is as follows: */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct hv_mmu_statistics { unsigned long immu_tsb_hits_ctx0_8k_tte; unsigned long immu_tsb_ticks_ctx0_8k_tte; @@ -2828,9 +3332,9 @@ struct hv_mmu_statistics { */ #define HV_FAST_MMUSTAT_INFO 0x103 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_mmustat_conf(unsigned long ra, unsigned long *orig_ra); -extern unsigned long sun4v_mmustat_info(unsigned long *ra); +#ifndef __ASSEMBLER__ +unsigned long sun4v_mmustat_conf(unsigned long ra, unsigned long *orig_ra); +unsigned long sun4v_mmustat_info(unsigned long *ra); #endif /* NCS crypto services */ @@ -2839,7 +3343,7 @@ extern unsigned long sun4v_mmustat_info(unsigned long *ra); #define HV_NCS_QCONF 0x01 #define HV_NCS_QTAIL_UPDATE 0x02 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct hv_ncs_queue_entry { /* MAU Control Register */ unsigned long mau_control; @@ -2918,10 +3422,10 @@ struct hv_ncs_qtail_update_arg { */ #define HV_FAST_NCS_REQUEST 0x110 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_ncs_request(unsigned long request, - unsigned long arg_ra, - unsigned long arg_size); +#ifndef __ASSEMBLER__ +unsigned long sun4v_ncs_request(unsigned long request, + unsigned long arg_ra, + unsigned long arg_size); #endif #define HV_FAST_FIRE_GET_PERFREG 0x120 @@ -2929,19 +3433,40 @@ extern unsigned long sun4v_ncs_request(unsigned long request, #define HV_FAST_REBOOT_DATA_SET 0x172 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_reboot_data_set(unsigned long ra, - unsigned long len); +#ifndef __ASSEMBLER__ +unsigned long sun4v_reboot_data_set(unsigned long ra, + unsigned long len); #endif #define HV_FAST_VT_GET_PERFREG 0x184 #define HV_FAST_VT_SET_PERFREG 0x185 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_vt_get_perfreg(unsigned long reg_num, - unsigned long *reg_val); -extern unsigned long sun4v_vt_set_perfreg(unsigned long reg_num, - unsigned long reg_val); +#ifndef __ASSEMBLER__ +unsigned long sun4v_vt_get_perfreg(unsigned long reg_num, + unsigned long *reg_val); +unsigned long sun4v_vt_set_perfreg(unsigned long reg_num, + unsigned long reg_val); +#endif + +#define HV_FAST_T5_GET_PERFREG 0x1a8 +#define HV_FAST_T5_SET_PERFREG 0x1a9 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_t5_get_perfreg(unsigned long reg_num, + unsigned long *reg_val); +unsigned long sun4v_t5_set_perfreg(unsigned long reg_num, + unsigned long reg_val); +#endif + + +#define HV_FAST_M7_GET_PERFREG 0x43 +#define HV_FAST_M7_SET_PERFREG 0x44 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_m7_get_perfreg(unsigned long reg_num, + unsigned long *reg_val); +unsigned long sun4v_m7_set_perfreg(unsigned long reg_num, + unsigned long reg_val); #endif /* Function numbers for HV_CORE_TRAP. */ @@ -2968,6 +3493,9 @@ extern unsigned long sun4v_vt_set_perfreg(unsigned long reg_num, #define HV_GRP_SDIO 0x0108 #define HV_GRP_SDIO_ERR 0x0109 #define HV_GRP_REBOOT_DATA 0x0110 +#define HV_GRP_ATU 0x0111 +#define HV_GRP_DAX 0x0113 +#define HV_GRP_M7_PERF 0x0114 #define HV_GRP_NIAG_PERF 0x0200 #define HV_GRP_FIRE_PERF 0x0201 #define HV_GRP_N2_CPU 0x0202 @@ -2975,24 +3503,25 @@ extern unsigned long sun4v_vt_set_perfreg(unsigned long reg_num, #define HV_GRP_VF_CPU 0x0205 #define HV_GRP_KT_CPU 0x0209 #define HV_GRP_VT_CPU 0x020c +#define HV_GRP_T5_CPU 0x0211 #define HV_GRP_DIAG 0x0300 -#ifndef __ASSEMBLY__ -extern unsigned long sun4v_get_version(unsigned long group, - unsigned long *major, - unsigned long *minor); -extern unsigned long sun4v_set_version(unsigned long group, - unsigned long major, - unsigned long minor, - unsigned long *actual_minor); - -extern int sun4v_hvapi_register(unsigned long group, unsigned long major, - unsigned long *minor); -extern void sun4v_hvapi_unregister(unsigned long group); -extern int sun4v_hvapi_get(unsigned long group, - unsigned long *major, - unsigned long *minor); -extern void sun4v_hvapi_init(void); +#ifndef __ASSEMBLER__ +unsigned long sun4v_get_version(unsigned long group, + unsigned long *major, + unsigned long *minor); +unsigned long sun4v_set_version(unsigned long group, + unsigned long major, + unsigned long minor, + unsigned long *actual_minor); + +int sun4v_hvapi_register(unsigned long group, unsigned long major, + unsigned long *minor); +void sun4v_hvapi_unregister(unsigned long group); +int sun4v_hvapi_get(unsigned long group, + unsigned long *major, + unsigned long *minor); +void sun4v_hvapi_init(void); #endif #endif /* !(_SPARC64_HYPERVISOR_H) */ diff --git a/arch/sparc/include/asm/ide.h b/arch/sparc/include/asm/ide.h deleted file mode 100644 index b7af3d658239..000000000000 --- a/arch/sparc/include/asm/ide.h +++ /dev/null @@ -1,97 +0,0 @@ -/* ide.h: SPARC PCI specific IDE glue. - * - * Copyright (C) 1997 David S. Miller (davem@davemloft.net) - * Copyright (C) 1998 Eddie C. Dost (ecd@skynet.be) - * Adaptation from sparc64 version to sparc by Pete Zaitcev. - */ - -#ifndef _SPARC_IDE_H -#define _SPARC_IDE_H - -#ifdef __KERNEL__ - -#include <asm/io.h> -#ifdef CONFIG_SPARC64 -#include <asm/pgalloc.h> -#include <asm/spitfire.h> -#include <asm/cacheflush.h> -#include <asm/page.h> -#else -#include <asm/pgtable.h> -#include <asm/psr.h> -#endif - -#define __ide_insl(data_reg, buffer, wcount) \ - __ide_insw(data_reg, buffer, (wcount)<<1) -#define __ide_outsl(data_reg, buffer, wcount) \ - __ide_outsw(data_reg, buffer, (wcount)<<1) - -/* On sparc, I/O ports and MMIO registers are accessed identically. */ -#define __ide_mm_insw __ide_insw -#define __ide_mm_insl __ide_insl -#define __ide_mm_outsw __ide_outsw -#define __ide_mm_outsl __ide_outsl - -static inline void __ide_insw(void __iomem *port, void *dst, u32 count) -{ -#if defined(CONFIG_SPARC64) && defined(DCACHE_ALIASING_POSSIBLE) - unsigned long end = (unsigned long)dst + (count << 1); -#endif - u16 *ps = dst; - u32 *pi; - - if(((unsigned long)ps) & 0x2) { - *ps++ = __raw_readw(port); - count--; - } - pi = (u32 *)ps; - while(count >= 2) { - u32 w; - - w = __raw_readw(port) << 16; - w |= __raw_readw(port); - *pi++ = w; - count -= 2; - } - ps = (u16 *)pi; - if(count) - *ps++ = __raw_readw(port); - -#if defined(CONFIG_SPARC64) && defined(DCACHE_ALIASING_POSSIBLE) - __flush_dcache_range((unsigned long)dst, end); -#endif -} - -static inline void __ide_outsw(void __iomem *port, const void *src, u32 count) -{ -#if defined(CONFIG_SPARC64) && defined(DCACHE_ALIASING_POSSIBLE) - unsigned long end = (unsigned long)src + (count << 1); -#endif - const u16 *ps = src; - const u32 *pi; - - if(((unsigned long)src) & 0x2) { - __raw_writew(*ps++, port); - count--; - } - pi = (const u32 *)ps; - while(count >= 2) { - u32 w; - - w = *pi++; - __raw_writew((w >> 16), port); - __raw_writew(w, port); - count -= 2; - } - ps = (const u16 *)pi; - if(count) - __raw_writew(*ps, port); - -#if defined(CONFIG_SPARC64) && defined(DCACHE_ALIASING_POSSIBLE) - __flush_dcache_range((unsigned long)src, end); -#endif -} - -#endif /* __KERNEL__ */ - -#endif /* _SPARC_IDE_H */ diff --git a/arch/sparc/include/asm/idprom.h b/arch/sparc/include/asm/idprom.h index 6976aa2439c6..4c372199cd70 100644 --- a/arch/sparc/include/asm/idprom.h +++ b/arch/sparc/include/asm/idprom.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * idprom.h: Macros and defines for idprom routines * @@ -20,6 +21,6 @@ struct idprom { }; extern struct idprom *idprom; -extern void idprom_init(void); +void idprom_init(void); #endif /* !(_SPARC_IDPROM_H) */ diff --git a/arch/sparc/include/asm/intr_queue.h b/arch/sparc/include/asm/intr_queue.h index 206077dedc2a..d61be2a3aac1 100644 --- a/arch/sparc/include/asm/intr_queue.h +++ b/arch/sparc/include/asm/intr_queue.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_INTR_QUEUE_H #define _SPARC64_INTR_QUEUE_H diff --git a/arch/sparc/include/asm/io-unit.h b/arch/sparc/include/asm/io-unit.h index 01ab2f613e91..8c38f5b9f927 100644 --- a/arch/sparc/include/asm/io-unit.h +++ b/arch/sparc/include/asm/io-unit.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* io-unit.h: Definitions for the sun4d IO-UNIT. * * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) @@ -6,8 +7,8 @@ #define _SPARC_IO_UNIT_H #include <linux/spinlock.h> +#include <linux/pgtable.h> #include <asm/page.h> -#include <asm/pgtable.h> /* The io-unit handles all virtual to physical address translations * that occur between the SBUS and physical memory. Access by @@ -43,7 +44,7 @@ struct iounit_struct { unsigned long bmap[(IOUNIT_DMA_SIZE >> (PAGE_SHIFT + 3)) / sizeof(unsigned long)]; spinlock_t lock; - iopte_t *page_table; + iopte_t __iomem *page_table; unsigned long rotor[3]; unsigned long limit[4]; }; diff --git a/arch/sparc/include/asm/io.h b/arch/sparc/include/asm/io.h index f6902cf3cbe9..2dad9be9ec75 100644 --- a/arch/sparc/include/asm/io.h +++ b/arch/sparc/include/asm/io.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_IO_H #define ___ASM_SPARC_IO_H #if defined(__sparc__) && defined(__arch64__) @@ -18,4 +19,6 @@ #define writel_be(__w, __addr) __raw_writel(__w, __addr) #define writew_be(__l, __addr) __raw_writew(__l, __addr) +#include <asm-generic/io.h> + #endif diff --git a/arch/sparc/include/asm/io_32.h b/arch/sparc/include/asm/io_32.h index c1acbd891cbc..549f0a72280d 100644 --- a/arch/sparc/include/asm/io_32.h +++ b/arch/sparc/include/asm/io_32.h @@ -1,192 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_IO_H #define __SPARC_IO_H #include <linux/kernel.h> -#include <linux/types.h> #include <linux/ioport.h> /* struct resource */ -#include <asm/page.h> /* IO address mapping routines need this */ -#include <asm-generic/pci_iomap.h> - -#define page_to_phys(page) (page_to_pfn(page) << PAGE_SHIFT) - -static inline u32 flip_dword (u32 l) -{ - return ((l&0xff)<<24) | (((l>>8)&0xff)<<16) | (((l>>16)&0xff)<<8)| ((l>>24)&0xff); -} - -static inline u16 flip_word (u16 w) -{ - return ((w&0xff) << 8) | ((w>>8)&0xff); -} +#define IO_SPACE_LIMIT 0xffffffff -#define mmiowb() +#define memset_io(d,c,sz) _memset_io(d,c,sz) +#define memcpy_fromio(d,s,sz) _memcpy_fromio(d,s,sz) +#define memcpy_toio(d,s,sz) _memcpy_toio(d,s,sz) /* - * Memory mapped I/O to PCI + * Bus number may be embedded in the higher bits of the physical address. + * This is why we have no bus number argument to ioremap(). */ +void __iomem *ioremap(phys_addr_t offset, size_t size); +void iounmap(volatile void __iomem *addr); -static inline u8 __raw_readb(const volatile void __iomem *addr) -{ - return *(__force volatile u8 *)addr; -} - -static inline u16 __raw_readw(const volatile void __iomem *addr) -{ - return *(__force volatile u16 *)addr; -} - -static inline u32 __raw_readl(const volatile void __iomem *addr) -{ - return *(__force volatile u32 *)addr; -} - -static inline void __raw_writeb(u8 b, volatile void __iomem *addr) -{ - *(__force volatile u8 *)addr = b; -} - -static inline void __raw_writew(u16 w, volatile void __iomem *addr) -{ - *(__force volatile u16 *)addr = w; -} +#include <asm-generic/io.h> -static inline void __raw_writel(u32 l, volatile void __iomem *addr) +static inline void _memset_io(volatile void __iomem *dst, + int c, __kernel_size_t n) { - *(__force volatile u32 *)addr = l; -} - -static inline u8 __readb(const volatile void __iomem *addr) -{ - return *(__force volatile u8 *)addr; -} + volatile void __iomem *d = dst; -static inline u16 __readw(const volatile void __iomem *addr) -{ - return flip_word(*(__force volatile u16 *)addr); + while (n--) { + writeb(c, d); + d++; + } } -static inline u32 __readl(const volatile void __iomem *addr) +static inline void _memcpy_fromio(void *dst, const volatile void __iomem *src, + __kernel_size_t n) { - return flip_dword(*(__force volatile u32 *)addr); -} + char *d = dst; -static inline void __writeb(u8 b, volatile void __iomem *addr) -{ - *(__force volatile u8 *)addr = b; + while (n--) { + char tmp = readb(src); + *d++ = tmp; + src++; + } } -static inline void __writew(u16 w, volatile void __iomem *addr) +static inline void _memcpy_toio(volatile void __iomem *dst, const void *src, + __kernel_size_t n) { - *(__force volatile u16 *)addr = flip_word(w); -} + const char *s = src; + volatile void __iomem *d = dst; -static inline void __writel(u32 l, volatile void __iomem *addr) -{ - *(__force volatile u32 *)addr = flip_dword(l); + while (n--) { + char tmp = *s++; + writeb(tmp, d); + d++; + } } -#define readb(__addr) __readb(__addr) -#define readw(__addr) __readw(__addr) -#define readl(__addr) __readl(__addr) -#define readb_relaxed(__addr) readb(__addr) -#define readw_relaxed(__addr) readw(__addr) -#define readl_relaxed(__addr) readl(__addr) - -#define writeb(__b, __addr) __writeb((__b),(__addr)) -#define writew(__w, __addr) __writew((__w),(__addr)) -#define writel(__l, __addr) __writel((__l),(__addr)) - -/* - * I/O space operations - * - * Arrangement on a Sun is somewhat complicated. - * - * First of all, we want to use standard Linux drivers - * for keyboard, PC serial, etc. These drivers think - * they access I/O space and use inb/outb. - * On the other hand, EBus bridge accepts PCI *memory* - * cycles and converts them into ISA *I/O* cycles. - * Ergo, we want inb & outb to generate PCI memory cycles. - * - * If we want to issue PCI *I/O* cycles, we do this - * with a low 64K fixed window in PCIC. This window gets - * mapped somewhere into virtual kernel space and we - * can use inb/outb again. - */ -#define inb_local(__addr) __readb((void __iomem *)(unsigned long)(__addr)) -#define inb(__addr) __readb((void __iomem *)(unsigned long)(__addr)) -#define inw(__addr) __readw((void __iomem *)(unsigned long)(__addr)) -#define inl(__addr) __readl((void __iomem *)(unsigned long)(__addr)) - -#define outb_local(__b, __addr) __writeb(__b, (void __iomem *)(unsigned long)(__addr)) -#define outb(__b, __addr) __writeb(__b, (void __iomem *)(unsigned long)(__addr)) -#define outw(__w, __addr) __writew(__w, (void __iomem *)(unsigned long)(__addr)) -#define outl(__l, __addr) __writel(__l, (void __iomem *)(unsigned long)(__addr)) - -#define inb_p(__addr) inb(__addr) -#define outb_p(__b, __addr) outb(__b, __addr) -#define inw_p(__addr) inw(__addr) -#define outw_p(__w, __addr) outw(__w, __addr) -#define inl_p(__addr) inl(__addr) -#define outl_p(__l, __addr) outl(__l, __addr) - -void outsb(unsigned long addr, const void *src, unsigned long cnt); -void outsw(unsigned long addr, const void *src, unsigned long cnt); -void outsl(unsigned long addr, const void *src, unsigned long cnt); -void insb(unsigned long addr, void *dst, unsigned long count); -void insw(unsigned long addr, void *dst, unsigned long count); -void insl(unsigned long addr, void *dst, unsigned long count); - -#define IO_SPACE_LIMIT 0xffffffff - /* * SBus accessors. * * SBus has only one, memory mapped, I/O space. * We do not need to flip bytes for SBus of course. */ -static inline u8 _sbus_readb(const volatile void __iomem *addr) +static inline u8 sbus_readb(const volatile void __iomem *addr) { return *(__force volatile u8 *)addr; } -static inline u16 _sbus_readw(const volatile void __iomem *addr) +static inline u16 sbus_readw(const volatile void __iomem *addr) { return *(__force volatile u16 *)addr; } -static inline u32 _sbus_readl(const volatile void __iomem *addr) +static inline u32 sbus_readl(const volatile void __iomem *addr) { return *(__force volatile u32 *)addr; } -static inline void _sbus_writeb(u8 b, volatile void __iomem *addr) +static inline void sbus_writeb(u8 b, volatile void __iomem *addr) { *(__force volatile u8 *)addr = b; } -static inline void _sbus_writew(u16 w, volatile void __iomem *addr) +static inline void sbus_writew(u16 w, volatile void __iomem *addr) { *(__force volatile u16 *)addr = w; } -static inline void _sbus_writel(u32 l, volatile void __iomem *addr) +static inline void sbus_writel(u32 l, volatile void __iomem *addr) { *(__force volatile u32 *)addr = l; } -/* - * The only reason for #define's is to hide casts to unsigned long. - */ -#define sbus_readb(__addr) _sbus_readb(__addr) -#define sbus_readw(__addr) _sbus_readw(__addr) -#define sbus_readl(__addr) _sbus_readl(__addr) -#define sbus_writeb(__b, __addr) _sbus_writeb(__b, __addr) -#define sbus_writew(__w, __addr) _sbus_writew(__w, __addr) -#define sbus_writel(__l, __addr) _sbus_writel(__l, __addr) - -static inline void sbus_memset_io(volatile void __iomem *__dst, int c, __kernel_size_t n) +static inline void sbus_memset_io(volatile void __iomem *__dst, int c, + __kernel_size_t n) { while(n--) { sbus_writeb(c, __dst); @@ -194,22 +101,9 @@ static inline void sbus_memset_io(volatile void __iomem *__dst, int c, __kernel_ } } -static inline void -_memset_io(volatile void __iomem *dst, int c, __kernel_size_t n) -{ - volatile void __iomem *d = dst; - - while (n--) { - writeb(c, d); - d++; - } -} - -#define memset_io(d,c,sz) _memset_io(d,c,sz) - -static inline void -_sbus_memcpy_fromio(void *dst, const volatile void __iomem *src, - __kernel_size_t n) +static inline void sbus_memcpy_fromio(void *dst, + const volatile void __iomem *src, + __kernel_size_t n) { char *d = dst; @@ -220,25 +114,9 @@ _sbus_memcpy_fromio(void *dst, const volatile void __iomem *src, } } -#define sbus_memcpy_fromio(d, s, sz) _sbus_memcpy_fromio(d, s, sz) - -static inline void -_memcpy_fromio(void *dst, const volatile void __iomem *src, __kernel_size_t n) -{ - char *d = dst; - - while (n--) { - char tmp = readb(src); - *d++ = tmp; - src++; - } -} - -#define memcpy_fromio(d,s,sz) _memcpy_fromio(d,s,sz) - -static inline void -_sbus_memcpy_toio(volatile void __iomem *dst, const void *src, - __kernel_size_t n) +static inline void sbus_memcpy_toio(volatile void __iomem *dst, + const void *src, + __kernel_size_t n) { const char *s = src; volatile void __iomem *d = dst; @@ -250,89 +128,13 @@ _sbus_memcpy_toio(volatile void __iomem *dst, const void *src, } } -#define sbus_memcpy_toio(d, s, sz) _sbus_memcpy_toio(d, s, sz) - -static inline void -_memcpy_toio(volatile void __iomem *dst, const void *src, __kernel_size_t n) -{ - const char *s = src; - volatile void __iomem *d = dst; - - while (n--) { - char tmp = *s++; - writeb(tmp, d); - d++; - } -} - -#define memcpy_toio(d,s,sz) _memcpy_toio(d,s,sz) - -#ifdef __KERNEL__ - -/* - * Bus number may be embedded in the higher bits of the physical address. - * This is why we have no bus number argument to ioremap(). - */ -extern void __iomem *ioremap(unsigned long offset, unsigned long size); -#define ioremap_nocache(X,Y) ioremap((X),(Y)) -#define ioremap_wc(X,Y) ioremap((X),(Y)) -extern void iounmap(volatile void __iomem *addr); - -#define ioread8(X) readb(X) -#define ioread16(X) readw(X) -#define ioread16be(X) __raw_readw(X) -#define ioread32(X) readl(X) -#define ioread32be(X) __raw_readl(X) -#define iowrite8(val,X) writeb(val,X) -#define iowrite16(val,X) writew(val,X) -#define iowrite16be(val,X) __raw_writew(val,X) -#define iowrite32(val,X) writel(val,X) -#define iowrite32be(val,X) __raw_writel(val,X) - -static inline void ioread8_rep(void __iomem *port, void *buf, unsigned long count) -{ - insb((unsigned long __force)port, buf, count); -} -static inline void ioread16_rep(void __iomem *port, void *buf, unsigned long count) -{ - insw((unsigned long __force)port, buf, count); -} - -static inline void ioread32_rep(void __iomem *port, void *buf, unsigned long count) -{ - insl((unsigned long __force)port, buf, count); -} - -static inline void iowrite8_rep(void __iomem *port, const void *buf, unsigned long count) -{ - outsb((unsigned long __force)port, buf, count); -} - -static inline void iowrite16_rep(void __iomem *port, const void *buf, unsigned long count) -{ - outsw((unsigned long __force)port, buf, count); -} - -static inline void iowrite32_rep(void __iomem *port, const void *buf, unsigned long count) -{ - outsl((unsigned long __force)port, buf, count); -} - /* Create a virtual mapping cookie for an IO port range */ -extern void __iomem *ioport_map(unsigned long port, unsigned int nr); -extern void ioport_unmap(void __iomem *); +void __iomem *ioport_map(unsigned long port, unsigned int nr); +void ioport_unmap(void __iomem *); /* Create a virtual mapping cookie for a PCI BAR (memory or IO) */ struct pci_dev; -extern void pci_iounmap(struct pci_dev *dev, void __iomem *); - -/* - * At the moment, we do not use CMOS_READ anywhere outside of rtc.c, - * so rtc_port is static in it. This should not change unless a new - * hardware pops up. - */ -#define RTC_PORT(x) (rtc_port + (x)) -#define RTC_ALWAYS_BCD 0 +void pci_iounmap(struct pci_dev *dev, void __iomem *); static inline int sbus_can_dma_64bit(void) { @@ -343,21 +145,9 @@ static inline int sbus_can_burst64(void) return 0; /* actually, sparc_cpu_model==sun4d */ } struct device; -extern void sbus_set_sbus64(struct device *, int); - -#endif +void sbus_set_sbus64(struct device *, int); #define __ARCH_HAS_NO_PAGE_ZERO_MAPPED 1 -/* - * Convert a physical pointer to a virtual kernel pointer for /dev/mem - * access - */ -#define xlate_dev_mem_ptr(p) __va(p) - -/* - * Convert a virtual cached pointer to an uncached pointer - */ -#define xlate_dev_kmem_ptr(p) p #endif /* !(__SPARC_IO_H) */ diff --git a/arch/sparc/include/asm/io_64.h b/arch/sparc/include/asm/io_64.h index 09b0b88aeb2a..d8ed296624af 100644 --- a/arch/sparc/include/asm/io_64.h +++ b/arch/sparc/include/asm/io_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC64_IO_H #define __SPARC64_IO_H @@ -8,127 +9,102 @@ #include <asm/page.h> /* IO address mapping routines need this */ #include <asm/asi.h> #include <asm-generic/pci_iomap.h> - -/* PC crapola... */ -#define __SLOW_DOWN_IO do { } while (0) -#define SLOW_DOWN_IO do { } while (0) +#define pci_iomap pci_iomap /* BIO layer definitions. */ extern unsigned long kern_base, kern_size; -#define page_to_phys(page) (page_to_pfn(page) << PAGE_SHIFT) -static inline u8 _inb(unsigned long addr) +/* __raw_{read,write}{b,w,l,q} uses direct access. + * Access the memory as big endian bypassing the cache + * by using ASI_PHYS_BYPASS_EC_E + */ +#define __raw_readb __raw_readb +static inline u8 __raw_readb(const volatile void __iomem *addr) { u8 ret; - __asm__ __volatile__("lduba\t[%1] %2, %0\t/* pci_inb */" + __asm__ __volatile__("lduba\t[%1] %2, %0\t/* pci_raw_readb */" : "=r" (ret) - : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) - : "memory"); + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); return ret; } -static inline u16 _inw(unsigned long addr) +#define __raw_readw __raw_readw +static inline u16 __raw_readw(const volatile void __iomem *addr) { u16 ret; - __asm__ __volatile__("lduha\t[%1] %2, %0\t/* pci_inw */" + __asm__ __volatile__("lduha\t[%1] %2, %0\t/* pci_raw_readw */" : "=r" (ret) - : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) - : "memory"); + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); return ret; } -static inline u32 _inl(unsigned long addr) +#define __raw_readl __raw_readl +static inline u32 __raw_readl(const volatile void __iomem *addr) { u32 ret; - __asm__ __volatile__("lduwa\t[%1] %2, %0\t/* pci_inl */" + __asm__ __volatile__("lduwa\t[%1] %2, %0\t/* pci_raw_readl */" : "=r" (ret) - : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) - : "memory"); + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); return ret; } -static inline void _outb(u8 b, unsigned long addr) -{ - __asm__ __volatile__("stba\t%r0, [%1] %2\t/* pci_outb */" - : /* no outputs */ - : "Jr" (b), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) - : "memory"); -} - -static inline void _outw(u16 w, unsigned long addr) -{ - __asm__ __volatile__("stha\t%r0, [%1] %2\t/* pci_outw */" - : /* no outputs */ - : "Jr" (w), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) - : "memory"); -} - -static inline void _outl(u32 l, unsigned long addr) +#define __raw_readq __raw_readq +static inline u64 __raw_readq(const volatile void __iomem *addr) { - __asm__ __volatile__("stwa\t%r0, [%1] %2\t/* pci_outl */" - : /* no outputs */ - : "Jr" (l), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) - : "memory"); -} - -#define inb(__addr) (_inb((unsigned long)(__addr))) -#define inw(__addr) (_inw((unsigned long)(__addr))) -#define inl(__addr) (_inl((unsigned long)(__addr))) -#define outb(__b, __addr) (_outb((u8)(__b), (unsigned long)(__addr))) -#define outw(__w, __addr) (_outw((u16)(__w), (unsigned long)(__addr))) -#define outl(__l, __addr) (_outl((u32)(__l), (unsigned long)(__addr))) - -#define inb_p(__addr) inb(__addr) -#define outb_p(__b, __addr) outb(__b, __addr) -#define inw_p(__addr) inw(__addr) -#define outw_p(__w, __addr) outw(__w, __addr) -#define inl_p(__addr) inl(__addr) -#define outl_p(__l, __addr) outl(__l, __addr) + u64 ret; -extern void outsb(unsigned long, const void *, unsigned long); -extern void outsw(unsigned long, const void *, unsigned long); -extern void outsl(unsigned long, const void *, unsigned long); -extern void insb(unsigned long, void *, unsigned long); -extern void insw(unsigned long, void *, unsigned long); -extern void insl(unsigned long, void *, unsigned long); + __asm__ __volatile__("ldxa\t[%1] %2, %0\t/* pci_raw_readq */" + : "=r" (ret) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); -static inline void ioread8_rep(void __iomem *port, void *buf, unsigned long count) -{ - insb((unsigned long __force)port, buf, count); -} -static inline void ioread16_rep(void __iomem *port, void *buf, unsigned long count) -{ - insw((unsigned long __force)port, buf, count); + return ret; } -static inline void ioread32_rep(void __iomem *port, void *buf, unsigned long count) +#define __raw_writeb __raw_writeb +static inline void __raw_writeb(u8 b, const volatile void __iomem *addr) { - insl((unsigned long __force)port, buf, count); + __asm__ __volatile__("stba\t%r0, [%1] %2\t/* pci_raw_writeb */" + : /* no outputs */ + : "Jr" (b), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); } -static inline void iowrite8_rep(void __iomem *port, const void *buf, unsigned long count) +#define __raw_writew __raw_writew +static inline void __raw_writew(u16 w, const volatile void __iomem *addr) { - outsb((unsigned long __force)port, buf, count); + __asm__ __volatile__("stha\t%r0, [%1] %2\t/* pci_raw_writew */" + : /* no outputs */ + : "Jr" (w), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); } -static inline void iowrite16_rep(void __iomem *port, const void *buf, unsigned long count) +#define __raw_writel __raw_writel +static inline void __raw_writel(u32 l, const volatile void __iomem *addr) { - outsw((unsigned long __force)port, buf, count); + __asm__ __volatile__("stwa\t%r0, [%1] %2\t/* pci_raw_writel */" + : /* no outputs */ + : "Jr" (l), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); } -static inline void iowrite32_rep(void __iomem *port, const void *buf, unsigned long count) +#define __raw_writeq __raw_writeq +static inline void __raw_writeq(u64 q, const volatile void __iomem *addr) { - outsl((unsigned long __force)port, buf, count); + __asm__ __volatile__("stxa\t%r0, [%1] %2\t/* pci_raw_writeq */" + : /* no outputs */ + : "Jr" (q), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); } -/* Memory functions, same as I/O accesses on Ultra. */ -static inline u8 _readb(const volatile void __iomem *addr) +/* Memory functions, same as I/O accesses on Ultra. + * Access memory as little endian bypassing + * the cache by using ASI_PHYS_BYPASS_EC_E_L + */ +#define readb readb +#define readb_relaxed readb +static inline u8 readb(const volatile void __iomem *addr) { u8 ret; __asm__ __volatile__("lduba\t[%1] %2, %0\t/* pci_readb */" @@ -138,7 +114,9 @@ static inline u8 _readb(const volatile void __iomem *addr) return ret; } -static inline u16 _readw(const volatile void __iomem *addr) +#define readw readw +#define readw_relaxed readw +static inline u16 readw(const volatile void __iomem *addr) { u16 ret; __asm__ __volatile__("lduha\t[%1] %2, %0\t/* pci_readw */" @@ -149,7 +127,9 @@ static inline u16 _readw(const volatile void __iomem *addr) return ret; } -static inline u32 _readl(const volatile void __iomem *addr) +#define readl readl +#define readl_relaxed readl +static inline u32 readl(const volatile void __iomem *addr) { u32 ret; __asm__ __volatile__("lduwa\t[%1] %2, %0\t/* pci_readl */" @@ -160,7 +140,9 @@ static inline u32 _readl(const volatile void __iomem *addr) return ret; } -static inline u64 _readq(const volatile void __iomem *addr) +#define readq readq +#define readq_relaxed readq +static inline u64 readq(const volatile void __iomem *addr) { u64 ret; __asm__ __volatile__("ldxa\t[%1] %2, %0\t/* pci_readq */" @@ -171,7 +153,9 @@ static inline u64 _readq(const volatile void __iomem *addr) return ret; } -static inline void _writeb(u8 b, volatile void __iomem *addr) +#define writeb writeb +#define writeb_relaxed writeb +static inline void writeb(u8 b, volatile void __iomem *addr) { __asm__ __volatile__("stba\t%r0, [%1] %2\t/* pci_writeb */" : /* no outputs */ @@ -179,7 +163,9 @@ static inline void _writeb(u8 b, volatile void __iomem *addr) : "memory"); } -static inline void _writew(u16 w, volatile void __iomem *addr) +#define writew writew +#define writew_relaxed writew +static inline void writew(u16 w, volatile void __iomem *addr) { __asm__ __volatile__("stha\t%r0, [%1] %2\t/* pci_writew */" : /* no outputs */ @@ -187,7 +173,9 @@ static inline void _writew(u16 w, volatile void __iomem *addr) : "memory"); } -static inline void _writel(u32 l, volatile void __iomem *addr) +#define writel writel +#define writel_relaxed writel +static inline void writel(u32 l, volatile void __iomem *addr) { __asm__ __volatile__("stwa\t%r0, [%1] %2\t/* pci_writel */" : /* no outputs */ @@ -195,7 +183,9 @@ static inline void _writel(u32 l, volatile void __iomem *addr) : "memory"); } -static inline void _writeq(u64 q, volatile void __iomem *addr) +#define writeq writeq +#define writeq_relaxed writeq +static inline void writeq(u64 q, volatile void __iomem *addr) { __asm__ __volatile__("stxa\t%r0, [%1] %2\t/* pci_writeq */" : /* no outputs */ @@ -203,100 +193,105 @@ static inline void _writeq(u64 q, volatile void __iomem *addr) : "memory"); } -#define readb(__addr) _readb(__addr) -#define readw(__addr) _readw(__addr) -#define readl(__addr) _readl(__addr) -#define readq(__addr) _readq(__addr) -#define readb_relaxed(__addr) _readb(__addr) -#define readw_relaxed(__addr) _readw(__addr) -#define readl_relaxed(__addr) _readl(__addr) -#define readq_relaxed(__addr) _readq(__addr) -#define writeb(__b, __addr) _writeb(__b, __addr) -#define writew(__w, __addr) _writew(__w, __addr) -#define writel(__l, __addr) _writel(__l, __addr) -#define writeq(__q, __addr) _writeq(__q, __addr) - -/* Now versions without byte-swapping. */ -static inline u8 _raw_readb(unsigned long addr) +#define inb inb +static inline u8 inb(unsigned long addr) { - u8 ret; - - __asm__ __volatile__("lduba\t[%1] %2, %0\t/* pci_raw_readb */" - : "=r" (ret) - : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + return readb((volatile void __iomem *)addr); +} - return ret; +#define inw inw +static inline u16 inw(unsigned long addr) +{ + return readw((volatile void __iomem *)addr); } -static inline u16 _raw_readw(unsigned long addr) +#define inl inl +static inline u32 inl(unsigned long addr) { - u16 ret; + return readl((volatile void __iomem *)addr); +} - __asm__ __volatile__("lduha\t[%1] %2, %0\t/* pci_raw_readw */" - : "=r" (ret) - : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); +#define outb outb +static inline void outb(u8 b, unsigned long addr) +{ + writeb(b, (volatile void __iomem *)addr); +} - return ret; +#define outw outw +static inline void outw(u16 w, unsigned long addr) +{ + writew(w, (volatile void __iomem *)addr); } -static inline u32 _raw_readl(unsigned long addr) +#define outl outl +static inline void outl(u32 l, unsigned long addr) { - u32 ret; + writel(l, (volatile void __iomem *)addr); +} - __asm__ __volatile__("lduwa\t[%1] %2, %0\t/* pci_raw_readl */" - : "=r" (ret) - : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); - return ret; -} +#define inb_p(__addr) inb(__addr) +#define outb_p(__b, __addr) outb(__b, __addr) +#define inw_p(__addr) inw(__addr) +#define outw_p(__w, __addr) outw(__w, __addr) +#define inl_p(__addr) inl(__addr) +#define outl_p(__l, __addr) outl(__l, __addr) -static inline u64 _raw_readq(unsigned long addr) +void outsb(unsigned long, const void *, unsigned long); +void outsw(unsigned long, const void *, unsigned long); +void outsl(unsigned long, const void *, unsigned long); +#define outsb outsb +#define outsw outsw +#define outsl outsl +void insb(unsigned long, void *, unsigned long); +void insw(unsigned long, void *, unsigned long); +void insl(unsigned long, void *, unsigned long); +#define insb insb +#define insw insw +#define insl insl + +static inline void readsb(const volatile void __iomem *port, void *buf, unsigned long count) { - u64 ret; - - __asm__ __volatile__("ldxa\t[%1] %2, %0\t/* pci_raw_readq */" - : "=r" (ret) - : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + insb((unsigned long __force)port, buf, count); +} +#define readsb readsb - return ret; +static inline void readsw(const volatile void __iomem *port, void *buf, unsigned long count) +{ + insw((unsigned long __force)port, buf, count); } +#define readsw readsw -static inline void _raw_writeb(u8 b, unsigned long addr) +static inline void readsl(const volatile void __iomem *port, void *buf, unsigned long count) { - __asm__ __volatile__("stba\t%r0, [%1] %2\t/* pci_raw_writeb */" - : /* no outputs */ - : "Jr" (b), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + insl((unsigned long __force)port, buf, count); } +#define readsl readsl -static inline void _raw_writew(u16 w, unsigned long addr) +static inline void writesb(void __iomem *port, const void *buf, unsigned long count) { - __asm__ __volatile__("stha\t%r0, [%1] %2\t/* pci_raw_writew */" - : /* no outputs */ - : "Jr" (w), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + outsb((unsigned long __force)port, buf, count); } +#define writesb writesb -static inline void _raw_writel(u32 l, unsigned long addr) +static inline void writesw(void __iomem *port, const void *buf, unsigned long count) { - __asm__ __volatile__("stwa\t%r0, [%1] %2\t/* pci_raw_writel */" - : /* no outputs */ - : "Jr" (l), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + outsw((unsigned long __force)port, buf, count); } +#define writesw writesw -static inline void _raw_writeq(u64 q, unsigned long addr) +static inline void writesl(void __iomem *port, const void *buf, unsigned long count) { - __asm__ __volatile__("stxa\t%r0, [%1] %2\t/* pci_raw_writeq */" - : /* no outputs */ - : "Jr" (q), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + outsl((unsigned long __force)port, buf, count); } +#define writesl writesl -#define __raw_readb(__addr) (_raw_readb((unsigned long)(__addr))) -#define __raw_readw(__addr) (_raw_readw((unsigned long)(__addr))) -#define __raw_readl(__addr) (_raw_readl((unsigned long)(__addr))) -#define __raw_readq(__addr) (_raw_readq((unsigned long)(__addr))) -#define __raw_writeb(__b, __addr) (_raw_writeb((u8)(__b), (unsigned long)(__addr))) -#define __raw_writew(__w, __addr) (_raw_writew((u16)(__w), (unsigned long)(__addr))) -#define __raw_writel(__l, __addr) (_raw_writel((u32)(__l), (unsigned long)(__addr))) -#define __raw_writeq(__q, __addr) (_raw_writeq((u64)(__q), (unsigned long)(__addr))) +#define ioread8_rep(p,d,l) readsb(p,d,l) +#define ioread16_rep(p,d,l) readsw(p,d,l) +#define ioread32_rep(p,d,l) readsl(p,d,l) +#define iowrite8_rep(p,d,l) writesb(p,d,l) +#define iowrite16_rep(p,d,l) writesw(p,d,l) +#define iowrite32_rep(p,d,l) writesl(p,d,l) /* Valid I/O Space regions are anywhere, because each PCI bus supported * can live in an arbitrary area of the physical address range. @@ -306,96 +301,47 @@ static inline void _raw_writeq(u64 q, unsigned long addr) /* Now, SBUS variants, only difference from PCI is that we do * not use little-endian ASIs. */ -static inline u8 _sbus_readb(const volatile void __iomem *addr) +static inline u8 sbus_readb(const volatile void __iomem *addr) { - u8 ret; - - __asm__ __volatile__("lduba\t[%1] %2, %0\t/* sbus_readb */" - : "=r" (ret) - : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E) - : "memory"); - - return ret; + return __raw_readb(addr); } -static inline u16 _sbus_readw(const volatile void __iomem *addr) +static inline u16 sbus_readw(const volatile void __iomem *addr) { - u16 ret; - - __asm__ __volatile__("lduha\t[%1] %2, %0\t/* sbus_readw */" - : "=r" (ret) - : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E) - : "memory"); - - return ret; + return __raw_readw(addr); } -static inline u32 _sbus_readl(const volatile void __iomem *addr) +static inline u32 sbus_readl(const volatile void __iomem *addr) { - u32 ret; - - __asm__ __volatile__("lduwa\t[%1] %2, %0\t/* sbus_readl */" - : "=r" (ret) - : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E) - : "memory"); - - return ret; + return __raw_readl(addr); } -static inline u64 _sbus_readq(const volatile void __iomem *addr) +static inline u64 sbus_readq(const volatile void __iomem *addr) { - u64 ret; - - __asm__ __volatile__("ldxa\t[%1] %2, %0\t/* sbus_readq */" - : "=r" (ret) - : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E) - : "memory"); - - return ret; + return __raw_readq(addr); } -static inline void _sbus_writeb(u8 b, volatile void __iomem *addr) +static inline void sbus_writeb(u8 b, volatile void __iomem *addr) { - __asm__ __volatile__("stba\t%r0, [%1] %2\t/* sbus_writeb */" - : /* no outputs */ - : "Jr" (b), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E) - : "memory"); + __raw_writeb(b, addr); } -static inline void _sbus_writew(u16 w, volatile void __iomem *addr) +static inline void sbus_writew(u16 w, volatile void __iomem *addr) { - __asm__ __volatile__("stha\t%r0, [%1] %2\t/* sbus_writew */" - : /* no outputs */ - : "Jr" (w), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E) - : "memory"); + __raw_writew(w, addr); } -static inline void _sbus_writel(u32 l, volatile void __iomem *addr) +static inline void sbus_writel(u32 l, volatile void __iomem *addr) { - __asm__ __volatile__("stwa\t%r0, [%1] %2\t/* sbus_writel */" - : /* no outputs */ - : "Jr" (l), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E) - : "memory"); + __raw_writel(l, addr); } -static inline void _sbus_writeq(u64 l, volatile void __iomem *addr) +static inline void sbus_writeq(u64 q, volatile void __iomem *addr) { - __asm__ __volatile__("stxa\t%r0, [%1] %2\t/* sbus_writeq */" - : /* no outputs */ - : "Jr" (l), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E) - : "memory"); + __raw_writeq(q, addr); } -#define sbus_readb(__addr) _sbus_readb(__addr) -#define sbus_readw(__addr) _sbus_readw(__addr) -#define sbus_readl(__addr) _sbus_readl(__addr) -#define sbus_readq(__addr) _sbus_readq(__addr) -#define sbus_writeb(__b, __addr) _sbus_writeb(__b, __addr) -#define sbus_writew(__w, __addr) _sbus_writew(__w, __addr) -#define sbus_writel(__l, __addr) _sbus_writel(__l, __addr) -#define sbus_writeq(__l, __addr) _sbus_writeq(__l, __addr) - -static inline void _sbus_memset_io(volatile void __iomem *dst, int c, __kernel_size_t n) +static inline void sbus_memset_io(volatile void __iomem *dst, int c, __kernel_size_t n) { while(n--) { sbus_writeb(c, dst); @@ -403,10 +349,7 @@ static inline void _sbus_memset_io(volatile void __iomem *dst, int c, __kernel_s } } -#define sbus_memset_io(d,c,sz) _sbus_memset_io(d,c,sz) - -static inline void -_memset_io(volatile void __iomem *dst, int c, __kernel_size_t n) +static inline void memset_io(volatile void __iomem *dst, int c, __kernel_size_t n) { volatile void __iomem *d = dst; @@ -415,12 +358,10 @@ _memset_io(volatile void __iomem *dst, int c, __kernel_size_t n) d++; } } +#define memset_io memset_io -#define memset_io(d,c,sz) _memset_io(d,c,sz) - -static inline void -_sbus_memcpy_fromio(void *dst, const volatile void __iomem *src, - __kernel_size_t n) +static inline void sbus_memcpy_fromio(void *dst, const volatile void __iomem *src, + __kernel_size_t n) { char *d = dst; @@ -431,10 +372,9 @@ _sbus_memcpy_fromio(void *dst, const volatile void __iomem *src, } } -#define sbus_memcpy_fromio(d, s, sz) _sbus_memcpy_fromio(d, s, sz) -static inline void -_memcpy_fromio(void *dst, const volatile void __iomem *src, __kernel_size_t n) +static inline void memcpy_fromio(void *dst, const volatile void __iomem *src, + __kernel_size_t n) { char *d = dst; @@ -444,12 +384,10 @@ _memcpy_fromio(void *dst, const volatile void __iomem *src, __kernel_size_t n) src++; } } +#define memcpy_fromio memcpy_fromio -#define memcpy_fromio(d,s,sz) _memcpy_fromio(d,s,sz) - -static inline void -_sbus_memcpy_toio(volatile void __iomem *dst, const void *src, - __kernel_size_t n) +static inline void sbus_memcpy_toio(volatile void __iomem *dst, const void *src, + __kernel_size_t n) { const char *s = src; volatile void __iomem *d = dst; @@ -461,10 +399,8 @@ _sbus_memcpy_toio(volatile void __iomem *dst, const void *src, } } -#define sbus_memcpy_toio(d, s, sz) _sbus_memcpy_toio(d, s, sz) - -static inline void -_memcpy_toio(volatile void __iomem *dst, const void *src, __kernel_size_t n) +static inline void memcpy_toio(volatile void __iomem *dst, const void *src, + __kernel_size_t n) { const char *s = src; volatile void __iomem *d = dst; @@ -475,10 +411,7 @@ _memcpy_toio(volatile void __iomem *dst, const void *src, __kernel_size_t n) d++; } } - -#define memcpy_toio(d,s,sz) _memcpy_toio(d,s,sz) - -#define mmiowb() +#define memcpy_toio memcpy_toio #ifdef __KERNEL__ @@ -490,31 +423,40 @@ static inline void __iomem *ioremap(unsigned long offset, unsigned long size) return (void __iomem *)offset; } -#define ioremap_nocache(X,Y) ioremap((X),(Y)) #define ioremap_wc(X,Y) ioremap((X),(Y)) +#define ioremap_wt(X,Y) ioremap((X),(Y)) +static inline void __iomem *ioremap_np(unsigned long offset, unsigned long size) +{ + return NULL; + +} +#define ioremap_np ioremap_np static inline void iounmap(volatile void __iomem *addr) { } -#define ioread8(X) readb(X) -#define ioread16(X) readw(X) -#define ioread16be(X) __raw_readw(X) -#define ioread32(X) readl(X) -#define ioread32be(X) __raw_readl(X) -#define iowrite8(val,X) writeb(val,X) -#define iowrite16(val,X) writew(val,X) -#define iowrite16be(val,X) __raw_writew(val,X) -#define iowrite32(val,X) writel(val,X) -#define iowrite32be(val,X) __raw_writel(val,X) +#define ioread8 readb +#define ioread16 readw +#define ioread16be __raw_readw +#define ioread32 readl +#define ioread32be __raw_readl +#define iowrite8 writeb +#define iowrite16 writew +#define iowrite16be __raw_writew +#define iowrite32 writel +#define iowrite32be __raw_writel /* Create a virtual mapping cookie for an IO port range */ -extern void __iomem *ioport_map(unsigned long port, unsigned int nr); -extern void ioport_unmap(void __iomem *); +void __iomem *ioport_map(unsigned long port, unsigned int nr); +void ioport_unmap(void __iomem *); +#define ioport_map ioport_map +#define ioport_unmap ioport_unmap /* Create a virtual mapping cookie for a PCI BAR (memory or IO) */ struct pci_dev; -extern void pci_iounmap(struct pci_dev *dev, void __iomem *); +void pci_iounmap(struct pci_dev *dev, void __iomem *); +#define pci_iounmap pci_iounmap static inline int sbus_can_dma_64bit(void) { @@ -525,18 +467,7 @@ static inline int sbus_can_burst64(void) return 1; } struct device; -extern void sbus_set_sbus64(struct device *, int); - -/* - * Convert a physical pointer to a virtual kernel pointer for /dev/mem - * access - */ -#define xlate_dev_mem_ptr(p) __va(p) - -/* - * Convert a virtual cached pointer to an uncached pointer - */ -#define xlate_dev_kmem_ptr(p) p +void sbus_set_sbus64(struct device *, int); #endif diff --git a/arch/sparc/include/asm/ioctls.h b/arch/sparc/include/asm/ioctls.h index 77413b7e3a18..c92ac780c39f 100644 --- a/arch/sparc/include/asm/ioctls.h +++ b/arch/sparc/include/asm/ioctls.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_SPARC_IOCTLS_H #define _ASM_SPARC_IOCTLS_H diff --git a/arch/sparc/include/asm/iommu-common.h b/arch/sparc/include/asm/iommu-common.h new file mode 100644 index 000000000000..802c90c79d1f --- /dev/null +++ b/arch/sparc/include/asm/iommu-common.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IOMMU_COMMON_H +#define _LINUX_IOMMU_COMMON_H + +#include <linux/spinlock_types.h> +#include <linux/device.h> +#include <asm/page.h> + +#define IOMMU_POOL_HASHBITS 4 +#define IOMMU_NR_POOLS (1 << IOMMU_POOL_HASHBITS) +#define IOMMU_ERROR_CODE (~(unsigned long) 0) + +struct iommu_pool { + unsigned long start; + unsigned long end; + unsigned long hint; + spinlock_t lock; +}; + +struct iommu_map_table { + unsigned long table_map_base; + unsigned long table_shift; + unsigned long nr_pools; + void (*lazy_flush)(struct iommu_map_table *); + unsigned long poolsize; + struct iommu_pool pools[IOMMU_NR_POOLS]; + u32 flags; +#define IOMMU_HAS_LARGE_POOL 0x00000001 +#define IOMMU_NO_SPAN_BOUND 0x00000002 +#define IOMMU_NEED_FLUSH 0x00000004 + struct iommu_pool large_pool; + unsigned long *map; +}; + +extern void iommu_tbl_pool_init(struct iommu_map_table *iommu, + unsigned long num_entries, + u32 table_shift, + void (*lazy_flush)(struct iommu_map_table *), + bool large_pool, u32 npools, + bool skip_span_boundary_check); + +extern unsigned long iommu_tbl_range_alloc(struct device *dev, + struct iommu_map_table *iommu, + unsigned long npages, + unsigned long *handle, + unsigned long mask, + unsigned int align_order); + +extern void iommu_tbl_range_free(struct iommu_map_table *iommu, + u64 dma_addr, unsigned long npages, + unsigned long entry); + +#endif diff --git a/arch/sparc/include/asm/iommu.h b/arch/sparc/include/asm/iommu.h index e650965b4a8d..37935cb34865 100644 --- a/arch/sparc/include/asm/iommu.h +++ b/arch/sparc/include/asm/iommu.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_IOMMU_H #define ___ASM_SPARC_IOMMU_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/iommu_32.h b/arch/sparc/include/asm/iommu_32.h index 70c589c05a10..af51cd5ea3c1 100644 --- a/arch/sparc/include/asm/iommu_32.h +++ b/arch/sparc/include/asm/iommu_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* iommu.h: Definitions for the sun4m IOMMU. * * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) @@ -99,7 +100,7 @@ struct iommu_regs { #define IOPTE_WAZ 0x00000001 /* Write as zeros */ struct iommu_struct { - struct iommu_regs *regs; + struct iommu_regs __iomem *regs; iopte_t *page_table; /* For convenience */ unsigned long start; /* First managed virtual address */ @@ -108,14 +109,14 @@ struct iommu_struct { struct bit_map usemap; }; -static inline void iommu_invalidate(struct iommu_regs *regs) +static inline void iommu_invalidate(struct iommu_regs __iomem *regs) { - regs->tlbflush = 0; + sbus_writel(0, ®s->tlbflush); } -static inline void iommu_invalidate_page(struct iommu_regs *regs, unsigned long ba) +static inline void iommu_invalidate_page(struct iommu_regs __iomem *regs, unsigned long ba) { - regs->pageflush = (ba & PAGE_MASK); + sbus_writel(ba & PAGE_MASK, ®s->pageflush); } #endif /* !(_SPARC_IOMMU_H) */ diff --git a/arch/sparc/include/asm/iommu_64.h b/arch/sparc/include/asm/iommu_64.h index caf798b56191..0ef6dedf747e 100644 --- a/arch/sparc/include/asm/iommu_64.h +++ b/arch/sparc/include/asm/iommu_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* iommu.h: Definitions for the sun5 IOMMU. * * Copyright (C) 1996, 1999, 2007 David S. Miller (davem@davemloft.net) @@ -16,6 +17,7 @@ #define IOPTE_WRITE 0x0000000000000002UL #define IOMMU_NUM_CTXS 4096 +#include <asm/iommu-common.h> struct iommu_arena { unsigned long *map; @@ -23,12 +25,39 @@ struct iommu_arena { unsigned int limit; }; +#define ATU_64_SPACE_SIZE 0x800000000 /* 32G */ + +/* Data structures for SPARC ATU architecture */ +struct atu_iotsb { + void *table; /* IOTSB table base virtual addr*/ + u64 ra; /* IOTSB table real addr */ + u64 dvma_size; /* ranges[3].size or OS slected 32G size */ + u64 dvma_base; /* ranges[3].base */ + u64 table_size; /* IOTSB table size */ + u64 page_size; /* IO PAGE size for IOTSB */ + u32 iotsb_num; /* tsbnum is same as iotsb_handle */ +}; + +struct atu_ranges { + u64 base; + u64 size; +}; + +struct atu { + struct atu_ranges *ranges; + struct atu_iotsb *iotsb; + struct iommu_map_table tbl; + u64 base; + u64 size; + u64 dma_addr_mask; +}; + struct iommu { + struct iommu_map_table tbl; + struct atu *atu; spinlock_t lock; - struct iommu_arena arena; - void (*flush_all)(struct iommu *); + u32 dma_addr_mask; iopte_t *page_table; - u32 page_table_map_base; unsigned long iommu_control; unsigned long iommu_tsbbase; unsigned long iommu_flush; @@ -40,7 +69,6 @@ struct iommu { unsigned long dummy_page_pa; unsigned long ctx_lowest_free; DECLARE_BITMAP(ctx_bitmap, IOMMU_NUM_CTXS); - u32 dma_addr_mask; }; struct strbuf { @@ -58,8 +86,8 @@ struct strbuf { volatile unsigned long __flushflag_buf[(64+(64-1)) / sizeof(long)]; }; -extern int iommu_table_init(struct iommu *iommu, int tsbsize, - u32 dma_offset, u32 dma_addr_mask, - int numa_node); +int iommu_table_init(struct iommu *iommu, int tsbsize, + u32 dma_offset, u32 dma_addr_mask, + int numa_node); #endif /* !(_SPARC64_IOMMU_H) */ diff --git a/arch/sparc/include/asm/irq.h b/arch/sparc/include/asm/irq.h index 3b44a6a14074..a2efc274dbb0 100644 --- a/arch/sparc/include/asm/irq.h +++ b/arch/sparc/include/asm/irq.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_IRQ_H #define ___ASM_SPARC_IRQ_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/irq_32.h b/arch/sparc/include/asm/irq_32.h index 2ae3acaeb1b3..6ee48321cbc2 100644 --- a/arch/sparc/include/asm/irq_32.h +++ b/arch/sparc/include/asm/irq_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* irq.h: IRQ registers on the Sparc. * * Copyright (C) 1995, 2007 David S. Miller (davem@davemloft.net) @@ -16,7 +17,7 @@ #define irq_canonicalize(irq) (irq) -extern void __init init_IRQ(void); +void __init sun4d_init_sbi_irq(void); #define NO_IRQ 0xffffffff diff --git a/arch/sparc/include/asm/irq_64.h b/arch/sparc/include/asm/irq_64.h index abf6afe82ca8..8c4c0c87f998 100644 --- a/arch/sparc/include/asm/irq_64.h +++ b/arch/sparc/include/asm/irq_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* irq.h: IRQ registers on the 64-bit Sparc. * * Copyright (C) 1996 David S. Miller (davem@davemloft.net) @@ -37,34 +38,30 @@ * * ino_bucket->irq allocation is made during {sun4v_,}build_irq(). */ -#define NR_IRQS 255 +#define NR_IRQS (2048) -extern void irq_install_pre_handler(int irq, - void (*func)(unsigned int, void *, void *), - void *arg1, void *arg2); +void irq_install_pre_handler(int irq, + void (*func)(unsigned int, void *, void *), + void *arg1, void *arg2); #define irq_canonicalize(irq) (irq) -extern unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap); -extern unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino); -extern unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino); -extern unsigned int sun4v_build_msi(u32 devhandle, unsigned int *irq_p, - unsigned int msi_devino_start, - unsigned int msi_devino_end); -extern void sun4v_destroy_msi(unsigned int irq); -extern unsigned int sun4u_build_msi(u32 portid, unsigned int *irq_p, - unsigned int msi_devino_start, - unsigned int msi_devino_end, - unsigned long imap_base, - unsigned long iclr_base); -extern void sun4u_destroy_msi(unsigned int irq); - -extern unsigned char irq_alloc(unsigned int dev_handle, - unsigned int dev_ino); -#ifdef CONFIG_PCI_MSI -extern void irq_free(unsigned int irq); -#endif - -extern void __init init_IRQ(void); -extern void fixup_irqs(void); +unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap); +unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino); +unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino); +unsigned int sun4v_build_msi(u32 devhandle, unsigned int *irq_p, + unsigned int msi_devino_start, + unsigned int msi_devino_end); +void sun4v_destroy_msi(unsigned int irq); +unsigned int sun4u_build_msi(u32 portid, unsigned int *irq_p, + unsigned int msi_devino_start, + unsigned int msi_devino_end, + unsigned long imap_base, + unsigned long iclr_base); +void sun4u_destroy_msi(unsigned int irq); + +unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino); +void irq_free(unsigned int irq); + +void fixup_irqs(void); static inline void set_softint(unsigned long bits) { @@ -89,12 +86,12 @@ static inline unsigned long get_softint(void) return retval; } -void arch_trigger_all_cpu_backtrace(void); -#define arch_trigger_all_cpu_backtrace arch_trigger_all_cpu_backtrace +void arch_trigger_cpumask_backtrace(const struct cpumask *mask, + int exclude_cpu); +#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace extern void *hardirq_stack[NR_CPUS]; extern void *softirq_stack[NR_CPUS]; -#define __ARCH_HAS_DO_SOFTIRQ #define NO_IRQ 0xffffffff diff --git a/arch/sparc/include/asm/irqflags.h b/arch/sparc/include/asm/irqflags.h index 1e138632bd3f..d1bdf6c398d1 100644 --- a/arch/sparc/include/asm/irqflags.h +++ b/arch/sparc/include/asm/irqflags.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_IRQFLAGS_H #define ___ASM_SPARC_IRQFLAGS_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/irqflags_32.h b/arch/sparc/include/asm/irqflags_32.h index e414c06615c1..f5f20774faac 100644 --- a/arch/sparc/include/asm/irqflags_32.h +++ b/arch/sparc/include/asm/irqflags_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * include/asm/irqflags.h * @@ -10,14 +11,14 @@ #ifndef _ASM_IRQFLAGS_H #define _ASM_IRQFLAGS_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> #include <asm/psr.h> -extern void arch_local_irq_restore(unsigned long); -extern unsigned long arch_local_irq_save(void); -extern void arch_local_irq_enable(void); +void arch_local_irq_restore(unsigned long); +unsigned long arch_local_irq_save(void); +void arch_local_irq_enable(void); static inline notrace unsigned long arch_local_save_flags(void) { @@ -42,6 +43,6 @@ static inline notrace bool arch_irqs_disabled(void) return arch_irqs_disabled_flags(arch_local_save_flags()); } -#endif /* (__ASSEMBLY__) */ +#endif /* (__ASSEMBLER__) */ #endif /* !(_ASM_IRQFLAGS_H) */ diff --git a/arch/sparc/include/asm/irqflags_64.h b/arch/sparc/include/asm/irqflags_64.h index 23cd27f6beb4..0071566c2c22 100644 --- a/arch/sparc/include/asm/irqflags_64.h +++ b/arch/sparc/include/asm/irqflags_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * include/asm/irqflags.h * @@ -12,7 +13,7 @@ #include <asm/pil.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ static inline notrace unsigned long arch_local_save_flags(void) { @@ -92,6 +93,6 @@ static inline notrace unsigned long arch_local_irq_save(void) return flags; } -#endif /* (__ASSEMBLY__) */ +#endif /* (__ASSEMBLER__) */ #endif /* !(_ASM_IRQFLAGS_H) */ diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h index 5080d16a832f..f49d1e6104e1 100644 --- a/arch/sparc/include/asm/jump_label.h +++ b/arch/sparc/include/asm/jump_label.h @@ -1,28 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_SPARC_JUMP_LABEL_H #define _ASM_SPARC_JUMP_LABEL_H -#ifdef __KERNEL__ +#ifndef __ASSEMBLER__ #include <linux/types.h> #define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { - asm goto("1:\n\t" - "nop\n\t" - "nop\n\t" - ".pushsection __jump_table, \"aw\"\n\t" - ".align 4\n\t" - ".word 1b, %l[l_yes], %c0\n\t" - ".popsection \n\t" - : : "i" (key) : : l_yes); + asm goto("1:\n\t" + "nop\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + return false; l_yes: return true; } -#endif /* __KERNEL__ */ +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm goto("1:\n\t" + "b %l[l_yes]\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} typedef u32 jump_label_t; @@ -32,4 +48,5 @@ struct jump_entry { jump_label_t key; }; +#endif /* __ASSEMBLER__ */ #endif diff --git a/arch/sparc/include/asm/kdebug.h b/arch/sparc/include/asm/kdebug.h index 8d12581ca386..830053f91da1 100644 --- a/arch/sparc/include/asm/kdebug.h +++ b/arch/sparc/include/asm/kdebug.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_KDEBUG_H #define ___ASM_SPARC_KDEBUG_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/kdebug_32.h b/arch/sparc/include/asm/kdebug_32.h index 1d0b240222ef..7627701a032c 100644 --- a/arch/sparc/include/asm/kdebug_32.h +++ b/arch/sparc/include/asm/kdebug_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * kdebug.h: Defines and definitions for debugging the Linux kernel * under various kernel debuggers. @@ -18,7 +19,7 @@ #define DEBUG_BP_TRAP 126 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* The debug vector is passed in %o1 at boot time. It is a pointer to * a structure in the debuggers address space. Here is its format. */ @@ -63,7 +64,7 @@ enum die_val { DIE_OOPS, }; -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ /* Some nice offset defines for assembler code. */ #define KDEBUG_ENTRY_OFF 0x0 diff --git a/arch/sparc/include/asm/kdebug_64.h b/arch/sparc/include/asm/kdebug_64.h index feb3578e12c4..89428bd4f94b 100644 --- a/arch/sparc/include/asm/kdebug_64.h +++ b/arch/sparc/include/asm/kdebug_64.h @@ -1,15 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_KDEBUG_H #define _SPARC64_KDEBUG_H struct pt_regs; -extern void bad_trap(struct pt_regs *, long); +void bad_trap(struct pt_regs *, long); /* Grossly misnamed. */ enum die_val { DIE_OOPS = 1, DIE_DEBUG, /* ta 0x70 */ DIE_DEBUG_2, /* ta 0x71 */ + DIE_BPT, /* ta 0x73 */ + DIE_SSTEP, /* ta 0x74 */ DIE_DIE, DIE_TRAP, DIE_TRAP_TL1, diff --git a/arch/sparc/include/asm/kgdb.h b/arch/sparc/include/asm/kgdb.h index b6ef301d05bf..deabe0279cd9 100644 --- a/arch/sparc/include/asm/kgdb.h +++ b/arch/sparc/include/asm/kgdb.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_KGDB_H #define _SPARC_KGDB_H @@ -30,7 +31,10 @@ enum regnames { #define NUMREGBYTES ((GDB_Y + 1) * 8) #endif -extern void arch_kgdb_breakpoint(void); +struct pt_regs; +asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs); + +void arch_kgdb_breakpoint(void); #define BREAK_INSTR_SIZE 4 #define CACHE_FLUSH_IS_SAFE 1 diff --git a/arch/sparc/include/asm/kmap_types.h b/arch/sparc/include/asm/kmap_types.h deleted file mode 100644 index aad21745fbb9..000000000000 --- a/arch/sparc/include/asm/kmap_types.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef _ASM_KMAP_TYPES_H -#define _ASM_KMAP_TYPES_H - -/* Dummy header just to define km_type. None of this - * is actually used on sparc. -DaveM - */ - -#include <asm-generic/kmap_types.h> - -#endif diff --git a/arch/sparc/include/asm/kprobes.h b/arch/sparc/include/asm/kprobes.h index 5879d71afdaa..aec742cd898f 100644 --- a/arch/sparc/include/asm/kprobes.h +++ b/arch/sparc/include/asm/kprobes.h @@ -1,13 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_KPROBES_H #define _SPARC64_KPROBES_H +#include <asm-generic/kprobes.h> + +#define BREAKPOINT_INSTRUCTION 0x91d02070 /* ta 0x70 */ +#define BREAKPOINT_INSTRUCTION_2 0x91d02071 /* ta 0x71 */ + +#ifdef CONFIG_KPROBES #include <linux/types.h> #include <linux/percpu.h> typedef u32 kprobe_opcode_t; -#define BREAKPOINT_INSTRUCTION 0x91d02070 /* ta 0x70 */ -#define BREAKPOINT_INSTRUCTION_2 0x91d02071 /* ta 0x71 */ #define MAX_INSN_SIZE 2 #define kretprobe_blacklist_size 0 @@ -19,7 +24,7 @@ do { flushi(&(p)->ainsn.insn[0]); \ flushi(&(p)->ainsn.insn[1]); \ } while (0) -void kretprobe_trampoline(void); +void __kretprobe_trampoline(void); /* Architecture specific copy of original instruction*/ struct arch_specific_insn { @@ -39,11 +44,12 @@ struct kprobe_ctlblk { unsigned long kprobe_status; unsigned long kprobe_orig_tnpc; unsigned long kprobe_orig_tstate_pil; - struct pt_regs jprobe_saved_regs; struct prev_kprobe prev_kprobe; }; -extern int kprobe_exceptions_notify(struct notifier_block *self, - unsigned long val, void *data); -extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr); +int kprobe_fault_handler(struct pt_regs *regs, int trapnr); +asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, + struct pt_regs *regs); + +#endif /* CONFIG_KPROBES */ #endif /* _SPARC64_KPROBES_H */ diff --git a/arch/sparc/include/asm/ldc.h b/arch/sparc/include/asm/ldc.h index bdb524a7b814..4294738d40be 100644 --- a/arch/sparc/include/asm/ldc.h +++ b/arch/sparc/include/asm/ldc.h @@ -1,18 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_LDC_H #define _SPARC64_LDC_H #include <asm/hypervisor.h> extern int ldom_domaining_enabled; -extern void ldom_set_var(const char *var, const char *value); -extern void ldom_reboot(const char *boot_command); -extern void ldom_power_off(void); +void ldom_set_var(const char *var, const char *value); +void ldom_reboot(const char *boot_command); +void ldom_power_off(void); /* The event handler will be evoked when link state changes * or data becomes available on the receive side. * * For non-RAW links, if the LDC_EVENT_RESET event arrives the - * driver should reset all of it's internal state and reinvoke + * driver should reset all of its internal state and reinvoke * ldc_connect() to try and bring the link up again. * * For RAW links, ldc_connect() is not used. Instead the driver @@ -48,33 +49,43 @@ struct ldc_channel_config { #define LDC_STATE_READY 0x03 #define LDC_STATE_CONNECTED 0x04 +#define LDC_PACKET_SIZE 64 + struct ldc_channel; /* Allocate state for a channel. */ -extern struct ldc_channel *ldc_alloc(unsigned long id, - const struct ldc_channel_config *cfgp, - void *event_arg); +struct ldc_channel *ldc_alloc(unsigned long id, + const struct ldc_channel_config *cfgp, + void *event_arg, + const char *name); /* Shut down and free state for a channel. */ -extern void ldc_free(struct ldc_channel *lp); +void ldc_free(struct ldc_channel *lp); /* Register TX and RX queues of the link with the hypervisor. */ -extern int ldc_bind(struct ldc_channel *lp, const char *name); +int ldc_bind(struct ldc_channel *lp); +void ldc_unbind(struct ldc_channel *lp); /* For non-RAW protocols we need to complete a handshake before * communication can proceed. ldc_connect() does that, if the * handshake completes successfully, an LDC_EVENT_UP event will * be sent up to the driver. */ -extern int ldc_connect(struct ldc_channel *lp); -extern int ldc_disconnect(struct ldc_channel *lp); +int ldc_connect(struct ldc_channel *lp); +int ldc_disconnect(struct ldc_channel *lp); + +int ldc_state(struct ldc_channel *lp); +void ldc_set_state(struct ldc_channel *lp, u8 state); +int ldc_mode(struct ldc_channel *lp); +void __ldc_print(struct ldc_channel *lp, const char *caller); +int ldc_rx_reset(struct ldc_channel *lp); -extern int ldc_state(struct ldc_channel *lp); +#define ldc_print(chan) __ldc_print(chan, __func__) /* Read and write operations. Only valid when the link is up. */ -extern int ldc_write(struct ldc_channel *lp, const void *buf, - unsigned int size); -extern int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size); +int ldc_write(struct ldc_channel *lp, const void *buf, + unsigned int size); +int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size); #define LDC_MAP_SHADOW 0x01 #define LDC_MAP_DIRECT 0x02 @@ -92,22 +103,22 @@ struct ldc_trans_cookie { }; struct scatterlist; -extern int ldc_map_sg(struct ldc_channel *lp, - struct scatterlist *sg, int num_sg, - struct ldc_trans_cookie *cookies, int ncookies, - unsigned int map_perm); +int ldc_map_sg(struct ldc_channel *lp, + struct scatterlist *sg, int num_sg, + struct ldc_trans_cookie *cookies, int ncookies, + unsigned int map_perm); -extern int ldc_map_single(struct ldc_channel *lp, - void *buf, unsigned int len, - struct ldc_trans_cookie *cookies, int ncookies, - unsigned int map_perm); +int ldc_map_single(struct ldc_channel *lp, + void *buf, unsigned int len, + struct ldc_trans_cookie *cookies, int ncookies, + unsigned int map_perm); -extern void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, - int ncookies); +void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, + int ncookies); -extern int ldc_copy(struct ldc_channel *lp, int copy_dir, - void *buf, unsigned int len, unsigned long offset, - struct ldc_trans_cookie *cookies, int ncookies); +int ldc_copy(struct ldc_channel *lp, int copy_dir, + void *buf, unsigned int len, unsigned long offset, + struct ldc_trans_cookie *cookies, int ncookies); static inline int ldc_get_dring_entry(struct ldc_channel *lp, void *buf, unsigned int len, @@ -127,12 +138,12 @@ static inline int ldc_put_dring_entry(struct ldc_channel *lp, return ldc_copy(lp, LDC_COPY_OUT, buf, len, offset, cookies, ncookies); } -extern void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len, - struct ldc_trans_cookie *cookies, - int *ncookies, unsigned int map_perm); +void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len, + struct ldc_trans_cookie *cookies, + int *ncookies, unsigned int map_perm); -extern void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, - unsigned int len, - struct ldc_trans_cookie *cookies, int ncookies); +void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, + unsigned int len, + struct ldc_trans_cookie *cookies, int ncookies); #endif /* _SPARC64_LDC_H */ diff --git a/arch/sparc/include/asm/leon.h b/arch/sparc/include/asm/leon.h index c2f6ff6d7a35..053a24b67aed 100644 --- a/arch/sparc/include/asm/leon.h +++ b/arch/sparc/include/asm/leon.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2004 Konrad Eisele (eiselekd@web.de,konrad@gaisler.com) Gaisler Research * Copyright (C) 2004 Stefan Holst (mail@s-holst.de) Uni-Stuttgart @@ -58,7 +59,7 @@ #define ASI_LEON3_SYSCTRL_CFG_SNOOPING (1 << 27) #define ASI_LEON3_SYSCTRL_CFG_SSIZE(c) (1 << ((c >> 20) & 0xf)) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* do a physical address bypass write, i.e. for 0x80000000 */ static inline void leon_store_reg(unsigned long paddr, unsigned long value) @@ -82,8 +83,8 @@ static inline unsigned long leon_load_reg(unsigned long paddr) #define LEON_BYPASS_LOAD_PA(x) leon_load_reg((unsigned long)(x)) #define LEON_BYPASS_STORE_PA(x, v) leon_store_reg((unsigned long)(x), (unsigned long)(v)) -extern void leon_switch_mm(void); -extern void leon_init_IRQ(void); +void leon_switch_mm(void); +void leon_init_IRQ(void); static inline unsigned long sparc_leon3_get_dcachecfg(void) { @@ -131,7 +132,7 @@ static inline int sparc_leon3_cpuid(void) return sparc_leon3_asr17() >> 28; } -#endif /*!__ASSEMBLY__*/ +#endif /*!__ASSEMBLER__*/ #ifdef CONFIG_SMP # define LEON3_IRQ_IPI_DEFAULT 13 @@ -193,17 +194,17 @@ static inline int sparc_leon3_cpuid(void) #define LEON2_CCR_DSETS_MASK 0x03000000UL #define LEON2_CFG_SSIZE_MASK 0x00007000UL -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct vm_area_struct; -extern unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr); -extern void leon_flush_icache_all(void); -extern void leon_flush_dcache_all(void); -extern void leon_flush_cache_all(void); -extern void leon_flush_tlb_all(void); +unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr); +void leon_flush_icache_all(void); +void leon_flush_dcache_all(void); +void leon_flush_cache_all(void); +void leon_flush_tlb_all(void); extern int leon_flush_during_switch; -extern int leon_flush_needed(void); -extern void leon_flush_pcache_all(struct vm_area_struct *vma, unsigned long page); +int leon_flush_needed(void); +void leon_flush_pcache_all(struct vm_area_struct *vma, unsigned long page); /* struct that hold LEON3 cache configuration registers */ struct leon3_cacheregs { @@ -217,29 +218,28 @@ struct leon3_cacheregs { struct device_node; struct task_struct; -extern unsigned int leon_build_device_irq(unsigned int real_irq, - irq_flow_handler_t flow_handler, - const char *name, int do_ack); -extern void leon_update_virq_handling(unsigned int virq, - irq_flow_handler_t flow_handler, - const char *name, int do_ack); -extern void leon_init_timers(void); -extern void leon_trans_init(struct device_node *dp); -extern void leon_node_init(struct device_node *dp, struct device_node ***nextp); -extern void init_leon(void); -extern void poke_leonsparc(void); -extern void leon3_getCacheRegs(struct leon3_cacheregs *regs); +unsigned int leon_build_device_irq(unsigned int real_irq, + irq_flow_handler_t flow_handler, + const char *name, int do_ack); +void leon_update_virq_handling(unsigned int virq, + irq_flow_handler_t flow_handler, + const char *name, int do_ack); +void leon_init_timers(void); +void leon_node_init(struct device_node *dp, struct device_node ***nextp); +void init_leon(void); +void poke_leonsparc(void); +void leon3_getCacheRegs(struct leon3_cacheregs *regs); extern int leon3_ticker_irq; #ifdef CONFIG_SMP -extern int leon_smp_nrcpus(void); -extern void leon_clear_profile_irq(int cpu); -extern void leon_smp_done(void); -extern void leon_boot_cpus(void); -extern int leon_boot_one_cpu(int i, struct task_struct *); +int leon_smp_nrcpus(void); +void leon_clear_profile_irq(int cpu); +void leon_smp_done(void); +void leon_boot_cpus(void); +int leon_boot_one_cpu(int i, struct task_struct *); void leon_init_smp(void); void leon_enable_irq_cpu(unsigned int irq_nr, unsigned int cpu); -extern irqreturn_t leon_percpu_timer_interrupt(int irq, void *unused); +irqreturn_t leon_percpu_timer_interrupt(int irq, void *unused); extern unsigned int smpleon_ipi[]; extern unsigned int linux_trap_ipi15_leon[]; @@ -247,11 +247,20 @@ extern int leon_ipi_irq; #endif /* CONFIG_SMP */ -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ /* macros used in leon_mm.c */ #define PFN(x) ((x) >> PAGE_SHIFT) #define _pfn_valid(pfn) ((pfn < last_valid_pfn) && (pfn >= PFN(phys_base))) #define _SRMMU_PTE_PMASK_LEON 0xffffffff +/* + * On LEON PCI Memory space is mapped 1:1 with physical address space. + * + * I/O space is located at low 64Kbytes in PCI I/O space. The I/O addresses + * are converted into CPU addresses to virtual addresses that are mapped with + * MMU to the PCI Host PCI I/O space window which are translated to the low + * 64Kbytes by the Host controller. + */ + #endif diff --git a/arch/sparc/include/asm/leon_amba.h b/arch/sparc/include/asm/leon_amba.h index 24ec48c3ff90..2ff5714d7a63 100644 --- a/arch/sparc/include/asm/leon_amba.h +++ b/arch/sparc/include/asm/leon_amba.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* *Copyright (C) 2004 Konrad Eisele (eiselekd@web.de,konrad@gaisler.com), Gaisler Research *Copyright (C) 2004 Stefan Holst (mail@s-holst.de), Uni-Stuttgart @@ -7,7 +8,7 @@ #ifndef LEON_AMBA_H_INCLUDE #define LEON_AMBA_H_INCLUDE -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct amba_prom_registers { unsigned int phys_addr; /* The physical address of this register */ @@ -88,7 +89,7 @@ struct amba_prom_registers { #define LEON3_GPTIMER_CONFIG_NRTIMERS(c) ((c)->config & 0x7) #define LEON3_GPTIMER_CTRL_ISPENDING(r) (((r)&LEON3_GPTIMER_CTRL_PENDING) ? 1 : 0) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct leon3_irqctrl_regs_map { u32 ilevel; @@ -188,7 +189,7 @@ extern int leon_debug_irqout; extern unsigned long leon3_gptimer_irq; extern unsigned int sparc_leon_eirq; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #define LEON3_IO_AREA 0xfff00000 #define LEON3_CONF_AREA 0xff000 diff --git a/arch/sparc/include/asm/leon_pci.h b/arch/sparc/include/asm/leon_pci.h index bfd3ab3092b5..484a56e81dcc 100644 --- a/arch/sparc/include/asm/leon_pci.h +++ b/arch/sparc/include/asm/leon_pci.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * asm/leon_pci.h * @@ -16,7 +17,7 @@ struct leon_pci_info { int (*map_irq)(const struct pci_dev *dev, u8 slot, u8 pin); }; -extern void leon_pci_init(struct platform_device *ofdev, - struct leon_pci_info *info); +void leon_pci_init(struct platform_device *ofdev, + struct leon_pci_info *info); #endif /* _ASM_LEON_PCI_H_ */ diff --git a/arch/sparc/include/asm/lsu.h b/arch/sparc/include/asm/lsu.h index 7190f8de90a0..154ae76c0244 100644 --- a/arch/sparc/include/asm/lsu.h +++ b/arch/sparc/include/asm/lsu.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_LSU_H #define _SPARC64_LSU_H diff --git a/arch/sparc/include/asm/machines.h b/arch/sparc/include/asm/machines.h index fd6ddb05d1b7..9f78f70c6f11 100644 --- a/arch/sparc/include/asm/machines.h +++ b/arch/sparc/include/asm/machines.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * machines.h: Defines for taking apart the machine type value in the * idprom and determining the kind of machine we are on. diff --git a/arch/sparc/include/asm/mbus.h b/arch/sparc/include/asm/mbus.h index 14128bcc5821..8b6dbe701b9b 100644 --- a/arch/sparc/include/asm/mbus.h +++ b/arch/sparc/include/asm/mbus.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * mbus.h: Various defines for MBUS modules. * diff --git a/arch/sparc/include/asm/mc146818rtc.h b/arch/sparc/include/asm/mc146818rtc.h index 67ed9e3a0235..07faf754eb71 100644 --- a/arch/sparc/include/asm/mc146818rtc.h +++ b/arch/sparc/include/asm/mc146818rtc.h @@ -1,5 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_MC146818RTC_H #define ___ASM_SPARC_MC146818RTC_H + +#include <linux/spinlock.h> + +extern spinlock_t rtc_lock; + #if defined(__sparc__) && defined(__arch64__) #include <asm/mc146818rtc_64.h> #else diff --git a/arch/sparc/include/asm/mc146818rtc_32.h b/arch/sparc/include/asm/mc146818rtc_32.h index fa7eac926582..d8fd75da8d49 100644 --- a/arch/sparc/include/asm/mc146818rtc_32.h +++ b/arch/sparc/include/asm/mc146818rtc_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Machine dependent access functions for RTC registers. */ diff --git a/arch/sparc/include/asm/mc146818rtc_64.h b/arch/sparc/include/asm/mc146818rtc_64.h index 7238d174e0e3..b1708a7e5735 100644 --- a/arch/sparc/include/asm/mc146818rtc_64.h +++ b/arch/sparc/include/asm/mc146818rtc_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Machine dependent access functions for RTC registers. */ diff --git a/arch/sparc/include/asm/mdesc.h b/arch/sparc/include/asm/mdesc.h index 139097f3a67b..ec31a06d04cc 100644 --- a/arch/sparc/include/asm/mdesc.h +++ b/arch/sparc/include/asm/mdesc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_MDESC_H #define _SPARC64_MDESC_H @@ -12,13 +13,14 @@ struct mdesc_handle; * the first argument to all of the operational calls that work * on mdescs. */ -extern struct mdesc_handle *mdesc_grab(void); -extern void mdesc_release(struct mdesc_handle *); +struct mdesc_handle *mdesc_grab(void); +void mdesc_release(struct mdesc_handle *); #define MDESC_NODE_NULL (~(u64)0) +#define MDESC_MAX_STR_LEN 256 -extern u64 mdesc_node_by_name(struct mdesc_handle *handle, - u64 from_node, const char *name); +u64 mdesc_node_by_name(struct mdesc_handle *handle, + u64 from_node, const char *name); #define mdesc_for_each_node_by_name(__hdl, __node, __name) \ for (__node = mdesc_node_by_name(__hdl, MDESC_NODE_NULL, __name); \ (__node) != MDESC_NODE_NULL; \ @@ -34,9 +36,9 @@ extern u64 mdesc_node_by_name(struct mdesc_handle *handle, * * These same rules apply to mdesc_node_name(). */ -extern const void *mdesc_get_property(struct mdesc_handle *handle, - u64 node, const char *name, int *lenp); -extern const char *mdesc_node_name(struct mdesc_handle *hp, u64 node); +const void *mdesc_get_property(struct mdesc_handle *handle, + u64 node, const char *name, int *lenp); +const char *mdesc_node_name(struct mdesc_handle *hp, u64 node); /* MD arc iteration, the standard sequence is: * @@ -50,31 +52,48 @@ extern const char *mdesc_node_name(struct mdesc_handle *hp, u64 node); #define MDESC_ARC_TYPE_FWD "fwd" #define MDESC_ARC_TYPE_BACK "back" -extern u64 mdesc_next_arc(struct mdesc_handle *handle, u64 from, - const char *arc_type); +u64 mdesc_next_arc(struct mdesc_handle *handle, u64 from, + const char *arc_type); #define mdesc_for_each_arc(__arc, __hdl, __node, __type) \ for (__arc = mdesc_next_arc(__hdl, __node, __type); \ (__arc) != MDESC_NODE_NULL; \ __arc = mdesc_next_arc(__hdl, __arc, __type)) -extern u64 mdesc_arc_target(struct mdesc_handle *hp, u64 arc); +u64 mdesc_arc_target(struct mdesc_handle *hp, u64 arc); -extern void mdesc_update(void); +void mdesc_update(void); struct mdesc_notifier_client { - void (*add)(struct mdesc_handle *handle, u64 node); - void (*remove)(struct mdesc_handle *handle, u64 node); - + void (*add)(struct mdesc_handle *handle, u64 node, + const char *node_name); + void (*remove)(struct mdesc_handle *handle, u64 node, + const char *node_name); const char *node_name; struct mdesc_notifier_client *next; }; -extern void mdesc_register_notifier(struct mdesc_notifier_client *client); +void mdesc_register_notifier(struct mdesc_notifier_client *client); + +union md_node_info { + struct vdev_port { + u64 id; /* id */ + u64 parent_cfg_hdl; /* parent config handle */ + const char *name; /* name (property) */ + } vdev_port; + struct ds_port { + u64 id; /* id */ + } ds_port; +}; + +u64 mdesc_get_node(struct mdesc_handle *hp, const char *node_name, + union md_node_info *node_info); +int mdesc_get_node_info(struct mdesc_handle *hp, u64 node, + const char *node_name, union md_node_info *node_info); -extern void mdesc_fill_in_cpu_data(cpumask_t *mask); -extern void mdesc_populate_present_mask(cpumask_t *mask); -extern void mdesc_get_page_sizes(cpumask_t *mask, unsigned long *pgsz_mask); +void mdesc_fill_in_cpu_data(cpumask_t *mask); +void mdesc_populate_present_mask(cpumask_t *mask); +void mdesc_get_page_sizes(cpumask_t *mask, unsigned long *pgsz_mask); -extern void sun4v_mdesc_init(void); +void sun4v_mdesc_init(void); #endif diff --git a/arch/sparc/include/asm/memctrl.h b/arch/sparc/include/asm/memctrl.h index 4065c56af7b6..6790ed608cbc 100644 --- a/arch/sparc/include/asm/memctrl.h +++ b/arch/sparc/include/asm/memctrl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_MEMCTRL_H #define _SPARC_MEMCTRL_H diff --git a/arch/sparc/include/asm/mman.h b/arch/sparc/include/asm/mman.h index 59bb5938d852..a8bae8ad243a 100644 --- a/arch/sparc/include/asm/mman.h +++ b/arch/sparc/include/asm/mman.h @@ -1,10 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_MMAN_H__ #define __SPARC_MMAN_H__ #include <uapi/asm/mman.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define arch_mmap_check(addr,len,flags) sparc_mmap_check(addr,len) int sparc_mmap_check(unsigned long addr, unsigned long len); -#endif + +#ifdef CONFIG_SPARC64 +#include <asm/adi_64.h> + +static inline void ipi_set_tstate_mcde(void *arg) +{ + struct mm_struct *mm = arg; + + /* Set TSTATE_MCDE for the task using address map that ADI has been + * enabled on if the task is running. If not, it will be set + * automatically at the next context switch + */ + if (current->mm == mm) { + struct pt_regs *regs; + + regs = task_pt_regs(current); + regs->tstate |= TSTATE_MCDE; + } +} + +#define arch_calc_vm_prot_bits(prot, pkey) sparc_calc_vm_prot_bits(prot) +static inline vm_flags_t sparc_calc_vm_prot_bits(unsigned long prot) +{ + if (adi_capable() && (prot & PROT_ADI)) { + struct pt_regs *regs; + + if (!current->mm->context.adi) { + regs = task_pt_regs(current); + regs->tstate |= TSTATE_MCDE; + current->mm->context.adi = true; + on_each_cpu_mask(mm_cpumask(current->mm), + ipi_set_tstate_mcde, current->mm, 0); + } + return VM_SPARC_ADI; + } else { + return 0; + } +} + +#define arch_validate_prot(prot, addr) sparc_validate_prot(prot, addr) +static inline int sparc_validate_prot(unsigned long prot, unsigned long addr) +{ + if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_SEM | PROT_ADI)) + return 0; + return 1; +} + +#define arch_validate_flags(vm_flags) arch_validate_flags(vm_flags) +/* arch_validate_flags() - Ensure combination of flags is valid for a + * VMA. + */ +static inline bool arch_validate_flags(vm_flags_t vm_flags) +{ + /* If ADI is being enabled on this VMA, check for ADI + * capability on the platform and ensure VMA is suitable + * for ADI + */ + if (vm_flags & VM_SPARC_ADI) { + if (!adi_capable()) + return false; + + /* ADI can not be enabled on PFN mapped pages */ + if (vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) + return false; + + /* Mergeable pages can become unmergeable + * if ADI is enabled on them even if they + * have identical data on them. This can be + * because ADI enabled pages with identical + * data may still not have identical ADI + * tags on them. Disallow ADI on mergeable + * pages. + */ + if (vm_flags & VM_MERGEABLE) + return false; + } + return true; +} +#endif /* CONFIG_SPARC64 */ + +#endif /* __ASSEMBLER__ */ #endif /* __SPARC_MMAN_H__ */ diff --git a/arch/sparc/include/asm/mmu.h b/arch/sparc/include/asm/mmu.h index 88fa313887db..286457e589a6 100644 --- a/arch/sparc/include/asm/mmu.h +++ b/arch/sparc/include/asm/mmu.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_MMU_H #define ___ASM_SPARC_MMU_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/mmu_32.h b/arch/sparc/include/asm/mmu_32.h index 6f056e535cf8..d5b220ca7f40 100644 --- a/arch/sparc/include/asm/mmu_32.h +++ b/arch/sparc/include/asm/mmu_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __MMU_H #define __MMU_H diff --git a/arch/sparc/include/asm/mmu_64.h b/arch/sparc/include/asm/mmu_64.h index 76092c4dd277..4eeb938f3e61 100644 --- a/arch/sparc/include/asm/mmu_64.h +++ b/arch/sparc/include/asm/mmu_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __MMU_H #define __MMU_H @@ -52,13 +53,13 @@ #define CTX_NR_MASK TAG_CONTEXT_BITS #define CTX_HW_MASK (CTX_NR_MASK | CTX_PGSZ_MASK) -#define CTX_FIRST_VERSION ((_AC(1,UL) << CTX_VERSION_SHIFT) + _AC(1,UL)) +#define CTX_FIRST_VERSION BIT(CTX_VERSION_SHIFT) #define CTX_VALID(__ctx) \ (!(((__ctx.sparc64_ctx_val) ^ tlb_context_cache) & CTX_VERSION_MASK)) #define CTX_HWBITS(__ctx) ((__ctx.sparc64_ctx_val) & CTX_HW_MASK) #define CTX_NRBITS(__ctx) ((__ctx.sparc64_ctx_val) & CTX_NR_MASK) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define TSB_ENTRY_ALIGNMENT 16 @@ -67,9 +68,9 @@ struct tsb { unsigned long pte; } __attribute__((aligned(TSB_ENTRY_ALIGNMENT))); -extern void __tsb_insert(unsigned long ent, unsigned long tag, unsigned long pte); -extern void tsb_flush(unsigned long ent, unsigned long tag); -extern void tsb_init(struct tsb *tsb, unsigned long size); +void __tsb_insert(unsigned long ent, unsigned long tag, unsigned long pte); +void tsb_flush(unsigned long ent, unsigned long tag); +void tsb_init(struct tsb *tsb, unsigned long size); struct tsb_config { struct tsb *tsb; @@ -89,16 +90,34 @@ struct tsb_config { #define MM_NUM_TSBS 1 #endif +/* ADI tags are stored when a page is swapped out and the storage for + * tags is allocated dynamically. There is a tag storage descriptor + * associated with each set of tag storage pages. Tag storage descriptors + * are allocated dynamically. Since kernel will allocate a full page for + * each tag storage descriptor, we can store up to + * PAGE_SIZE/sizeof(tag storage descriptor) descriptors on that page. + */ +typedef struct { + unsigned long start; /* Start address for this tag storage */ + unsigned long end; /* Last address for tag storage */ + unsigned char *tags; /* Where the tags are */ + unsigned long tag_users; /* number of references to descriptor */ +} tag_storage_desc_t; + typedef struct { spinlock_t lock; unsigned long sparc64_ctx_val; - unsigned long huge_pte_count; - struct page *pgtable_page; + unsigned long hugetlb_pte_count; + unsigned long thp_pte_count; struct tsb_config tsb_block[MM_NUM_TSBS]; struct hv_tsb_descr tsb_descr[MM_NUM_TSBS]; + void *vdso; + bool adi; + tag_storage_desc_t *tag_store; + spinlock_t tag_lock; } mm_context_t; -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #define TSB_CONFIG_TSB 0x00 #define TSB_CONFIG_RSS_LIMIT 0x08 diff --git a/arch/sparc/include/asm/mmu_context.h b/arch/sparc/include/asm/mmu_context.h index 5531346c64f9..5e1ea38511f4 100644 --- a/arch/sparc/include/asm/mmu_context.h +++ b/arch/sparc/include/asm/mmu_context.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_MMU_CONTEXT_H #define ___ASM_SPARC_MMU_CONTEXT_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/mmu_context_32.h b/arch/sparc/include/asm/mmu_context_32.h index 2df2a9be8f6d..d9ff73f776f9 100644 --- a/arch/sparc/include/asm/mmu_context_32.h +++ b/arch/sparc/include/asm/mmu_context_32.h @@ -1,17 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_MMU_CONTEXT_H #define __SPARC_MMU_CONTEXT_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm-generic/mm_hooks.h> -static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) -{ -} - /* Initialize a new mmu context. This is invoked when a new * address space instance (unique or shared) is instantiated. */ +#define init_new_context init_new_context int init_new_context(struct task_struct *tsk, struct mm_struct *mm); /* Destroy a dead context. This occurs when mmput drops the @@ -19,17 +17,18 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm); * all the page tables have been flushed. Our job is to destroy * any remaining processor-specific state. */ +#define destroy_context destroy_context void destroy_context(struct mm_struct *mm); /* Switch the current MM context. */ void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, struct task_struct *tsk); -#define deactivate_mm(tsk,mm) do { } while (0) - /* Activate a new MM instance for the current task. */ #define activate_mm(active_mm, mm) switch_mm((active_mm), (mm), NULL) -#endif /* !(__ASSEMBLY__) */ +#include <asm-generic/mmu_context.h> + +#endif /* !(__ASSEMBLER__) */ #endif /* !(__SPARC_MMU_CONTEXT_H) */ diff --git a/arch/sparc/include/asm/mmu_context_64.h b/arch/sparc/include/asm/mmu_context_64.h index 3d528f06e4b0..78bbacc14d2d 100644 --- a/arch/sparc/include/asm/mmu_context_64.h +++ b/arch/sparc/include/asm/mmu_context_64.h @@ -1,54 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC64_MMU_CONTEXT_H #define __SPARC64_MMU_CONTEXT_H /* Derived heavily from Linus's Alpha/AXP ASN code... */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/spinlock.h> +#include <linux/mm_types.h> +#include <linux/smp.h> +#include <linux/sched.h> + #include <asm/spitfire.h> +#include <asm/adi_64.h> #include <asm-generic/mm_hooks.h> - -static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) -{ -} +#include <asm/percpu.h> extern spinlock_t ctx_alloc_lock; extern unsigned long tlb_context_cache; extern unsigned long mmu_context_bmap[]; -extern void get_new_mmu_context(struct mm_struct *mm); -#ifdef CONFIG_SMP -extern void smp_new_mmu_context_version(void); -#else -#define smp_new_mmu_context_version() do { } while (0) -#endif +DECLARE_PER_CPU(struct mm_struct *, per_cpu_secondary_mm); +void get_new_mmu_context(struct mm_struct *mm); -extern int init_new_context(struct task_struct *tsk, struct mm_struct *mm); -extern void destroy_context(struct mm_struct *mm); +#define init_new_context init_new_context +int init_new_context(struct task_struct *tsk, struct mm_struct *mm); +#define destroy_context destroy_context +void destroy_context(struct mm_struct *mm); -extern void __tsb_context_switch(unsigned long pgd_pa, - struct tsb_config *tsb_base, - struct tsb_config *tsb_huge, - unsigned long tsb_descr_pa); +void __tsb_context_switch(unsigned long pgd_pa, + struct tsb_config *tsb_base, + struct tsb_config *tsb_huge, + unsigned long tsb_descr_pa, + unsigned long secondary_ctx); -static inline void tsb_context_switch(struct mm_struct *mm) +static inline void tsb_context_switch_ctx(struct mm_struct *mm, + unsigned long ctx) { __tsb_context_switch(__pa(mm->pgd), - &mm->context.tsb_block[0], + &mm->context.tsb_block[MM_TSB_BASE], #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - (mm->context.tsb_block[1].tsb ? - &mm->context.tsb_block[1] : + (mm->context.tsb_block[MM_TSB_HUGE].tsb ? + &mm->context.tsb_block[MM_TSB_HUGE] : NULL) #else NULL #endif - , __pa(&mm->context.tsb_descr[0])); + , __pa(&mm->context.tsb_descr[MM_TSB_BASE]), + ctx); } -extern void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long mm_rss); +#define tsb_context_switch(X) tsb_context_switch_ctx(X, 0) + +void tsb_grow(struct mm_struct *mm, + unsigned long tsb_index, + unsigned long mm_rss); #ifdef CONFIG_SMP -extern void smp_tsb_sync(struct mm_struct *mm); +void smp_tsb_sync(struct mm_struct *mm); #else #define smp_tsb_sync(__mm) do { } while (0) #endif @@ -66,14 +74,15 @@ extern void smp_tsb_sync(struct mm_struct *mm); : "r" (CTX_HWBITS((__mm)->context)), \ "r" (SECONDARY_CONTEXT), "i" (ASI_DMMU), "i" (ASI_MMU)) -extern void __flush_tlb_mm(unsigned long, unsigned long); +void __flush_tlb_mm(unsigned long, unsigned long); /* Switch the current MM context. */ static inline void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, struct task_struct *tsk) { unsigned long ctx_valid, flags; - int cpu; + int cpu = smp_processor_id(); + per_cpu(per_cpu_secondary_mm, cpu) = mm; if (unlikely(mm == &init_mm)) return; @@ -84,7 +93,7 @@ static inline void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, str /* We have to be extremely careful here or else we will miss * a TSB grow if we switch back and forth between a kernel - * thread and an address space which has it's TSB size increased + * thread and an address space which has its TSB size increased * on another processor. * * It is possible to play some games in order to optimize the @@ -109,17 +118,15 @@ static inline void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, str * * At that point cpu0 continues to use a stale TSB, the one from * before the TSB grow performed on cpu1. cpu1 did not cross-call - * cpu0 to update it's TSB because at that point the cpu_vm_mask + * cpu0 to update its TSB because at that point the cpu_vm_mask * only had cpu1 set in it. */ - load_secondary_context(mm); - tsb_context_switch(mm); + tsb_context_switch_ctx(mm, CTX_HWBITS(mm->context)); /* Any time a processor runs a context on an address space * for the first time, we must flush that context out of the * local TLB. */ - cpu = smp_processor_id(); if (!ctx_valid || !cpumask_test_cpu(cpu, mm_cpumask(mm))) { cpumask_set_cpu(cpu, mm_cpumask(mm)); __flush_tlb_mm(CTX_HWBITS(mm->context), @@ -128,27 +135,64 @@ static inline void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, str spin_unlock_irqrestore(&mm->context.lock, flags); } -#define deactivate_mm(tsk,mm) do { } while (0) +#define activate_mm(active_mm, mm) switch_mm(active_mm, mm, NULL) -/* Activate a new MM instance for the current task. */ -static inline void activate_mm(struct mm_struct *active_mm, struct mm_struct *mm) +#define __HAVE_ARCH_START_CONTEXT_SWITCH +static inline void arch_start_context_switch(struct task_struct *prev) { - unsigned long flags; - int cpu; + /* Save the current state of MCDPER register for the process + * we are switching from + */ + if (adi_capable()) { + register unsigned long tmp_mcdper; + + __asm__ __volatile__( + ".word 0x83438000\n\t" /* rd %mcdper, %g1 */ + "mov %%g1, %0\n\t" + : "=r" (tmp_mcdper) + : + : "g1"); + if (tmp_mcdper) + set_tsk_thread_flag(prev, TIF_MCDPER); + else + clear_tsk_thread_flag(prev, TIF_MCDPER); + } +} - spin_lock_irqsave(&mm->context.lock, flags); - if (!CTX_VALID(mm->context)) - get_new_mmu_context(mm); - cpu = smp_processor_id(); - if (!cpumask_test_cpu(cpu, mm_cpumask(mm))) - cpumask_set_cpu(cpu, mm_cpumask(mm)); +#define finish_arch_post_lock_switch finish_arch_post_lock_switch +static inline void finish_arch_post_lock_switch(void) +{ + /* Restore the state of MCDPER register for the new process + * just switched to. + */ + if (adi_capable()) { + register unsigned long tmp_mcdper; + + tmp_mcdper = test_thread_flag(TIF_MCDPER); + __asm__ __volatile__( + "mov %0, %%g1\n\t" + ".word 0x9d800001\n\t" /* wr %g0, %g1, %mcdper" */ + ".word 0xaf902001\n\t" /* wrpr %g0, 1, %pmcdper */ + : + : "ir" (tmp_mcdper) + : "g1"); + if (current && current->mm && current->mm->context.adi) { + struct pt_regs *regs; + + regs = task_pt_regs(current); + regs->tstate |= TSTATE_MCDE; + } + } +} - load_secondary_context(mm); - __flush_tlb_mm(CTX_HWBITS(mm->context), SECONDARY_CONTEXT); - tsb_context_switch(mm); - spin_unlock_irqrestore(&mm->context.lock, flags); +#define mm_untag_mask mm_untag_mask +static inline unsigned long mm_untag_mask(struct mm_struct *mm) +{ + return -1UL >> adi_nbits(); } -#endif /* !(__ASSEMBLY__) */ +#include <asm-generic/mmu_context.h> + +#endif /* !(__ASSEMBLER__) */ #endif /* !(__SPARC64_MMU_CONTEXT_H) */ diff --git a/arch/sparc/include/asm/mmzone.h b/arch/sparc/include/asm/mmzone.h index 99d9b9f577bf..74eb2c71d077 100644 --- a/arch/sparc/include/asm/mmzone.h +++ b/arch/sparc/include/asm/mmzone.h @@ -1,17 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_MMZONE_H #define _SPARC64_MMZONE_H -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA #include <linux/cpumask.h> -extern struct pglist_data *node_data[]; - -#define NODE_DATA(nid) (node_data[nid]) - extern int numa_cpu_lookup_table[]; extern cpumask_t numa_cpumask_lookup_table[]; -#endif /* CONFIG_NEED_MULTIPLE_NODES */ +#endif /* CONFIG_NUMA */ #endif /* _SPARC64_MMZONE_H */ diff --git a/arch/sparc/include/asm/msi.h b/arch/sparc/include/asm/msi.h deleted file mode 100644 index 724ca5667052..000000000000 --- a/arch/sparc/include/asm/msi.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * msi.h: Defines specific to the MBus - Sbus - Interface. - * - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) - */ - -#ifndef _SPARC_MSI_H -#define _SPARC_MSI_H - -/* - * Locations of MSI Registers. - */ -#define MSI_MBUS_ARBEN 0xe0001008 /* MBus Arbiter Enable register */ - -/* - * Useful bits in the MSI Registers. - */ -#define MSI_ASYNC_MODE 0x80000000 /* Operate the MSI asynchronously */ - - -static inline void msi_set_sync(void) -{ - __asm__ __volatile__ ("lda [%0] %1, %%g3\n\t" - "andn %%g3, %2, %%g3\n\t" - "sta %%g3, [%0] %1\n\t" : : - "r" (MSI_MBUS_ARBEN), - "i" (ASI_M_CTL), "r" (MSI_ASYNC_MODE) : "g3"); -} - -#endif /* !(_SPARC_MSI_H) */ diff --git a/arch/sparc/include/asm/mxcc.h b/arch/sparc/include/asm/mxcc.h index c0517bd05bde..bd6339dcf693 100644 --- a/arch/sparc/include/asm/mxcc.h +++ b/arch/sparc/include/asm/mxcc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * mxcc.h: Definitions of the Viking MXCC registers * @@ -83,7 +84,7 @@ * MID: The moduleID of the cpu your read this from. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ static inline void mxcc_set_stream_src(unsigned long *paddr) { @@ -132,6 +133,6 @@ static inline void mxcc_set_creg(unsigned long mxcc_control) "i" (ASI_M_MXCC)); } -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* !(_SPARC_MXCC_H) */ diff --git a/arch/sparc/include/asm/nmi.h b/arch/sparc/include/asm/nmi.h index 72e6500e7ab0..920dc23f443f 100644 --- a/arch/sparc/include/asm/nmi.h +++ b/arch/sparc/include/asm/nmi.h @@ -1,13 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NMI_H #define __NMI_H -extern int __init nmi_init(void); -extern void perfctr_irq(int irq, struct pt_regs *regs); -extern void nmi_adjust_hz(unsigned int new_hz); +int __init nmi_init(void); +void perfctr_irq(int irq, struct pt_regs *regs); +void nmi_adjust_hz(unsigned int new_hz); extern atomic_t nmi_active; -extern void start_nmi_watchdog(void *unused); -extern void stop_nmi_watchdog(void *unused); +void start_nmi_watchdog(void *unused); +void stop_nmi_watchdog(void *unused); #endif /* __NMI_H */ diff --git a/arch/sparc/include/asm/ns87303.h b/arch/sparc/include/asm/ns87303.h index 6b947ee0f6aa..5401894a6065 100644 --- a/arch/sparc/include/asm/ns87303.h +++ b/arch/sparc/include/asm/ns87303.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* ns87303.h: Configuration Register Description for the * National Semiconductor PC87303 (SuperIO). * diff --git a/arch/sparc/include/asm/obio.h b/arch/sparc/include/asm/obio.h index 910c1d9af1f8..f1ad7f7bcac2 100644 --- a/arch/sparc/include/asm/obio.h +++ b/arch/sparc/include/asm/obio.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * obio.h: Some useful locations in 0xFXXXXXXXX PA obio space on sun4d. * @@ -96,7 +97,7 @@ #define CC_EREG 0x1F00E00 /* Error code register */ #define CC_CID 0x1F00F04 /* Component ID */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ static inline int bw_get_intr_mask(int sbus_level) { @@ -117,9 +118,9 @@ static inline void bw_clear_intr_mask(int sbus_level, int mask) "i" (ASI_M_CTL)); } -static inline unsigned bw_get_prof_limit(int cpu) +static inline unsigned int bw_get_prof_limit(int cpu) { - unsigned limit; + unsigned int limit; __asm__ __volatile__ ("lda [%1] %2, %0" : "=r" (limit) : @@ -128,7 +129,7 @@ static inline unsigned bw_get_prof_limit(int cpu) return limit; } -static inline void bw_set_prof_limit(int cpu, unsigned limit) +static inline void bw_set_prof_limit(int cpu, unsigned int limit) { __asm__ __volatile__ ("sta %0, [%1] %2" : : "r" (limit), @@ -136,9 +137,9 @@ static inline void bw_set_prof_limit(int cpu, unsigned limit) "i" (ASI_M_CTL)); } -static inline unsigned bw_get_ctrl(int cpu) +static inline unsigned int bw_get_ctrl(int cpu) { - unsigned ctrl; + unsigned int ctrl; __asm__ __volatile__ ("lda [%1] %2, %0" : "=r" (ctrl) : @@ -147,7 +148,7 @@ static inline unsigned bw_get_ctrl(int cpu) return ctrl; } -static inline void bw_set_ctrl(int cpu, unsigned ctrl) +static inline void bw_set_ctrl(int cpu, unsigned int ctrl) { __asm__ __volatile__ ("sta %0, [%1] %2" : : "r" (ctrl), @@ -155,9 +156,9 @@ static inline void bw_set_ctrl(int cpu, unsigned ctrl) "i" (ASI_M_CTL)); } -static inline unsigned cc_get_ipen(void) +static inline unsigned int cc_get_ipen(void) { - unsigned pending; + unsigned int pending; __asm__ __volatile__ ("lduha [%1] %2, %0" : "=r" (pending) : @@ -166,7 +167,7 @@ static inline unsigned cc_get_ipen(void) return pending; } -static inline void cc_set_iclr(unsigned clear) +static inline void cc_set_iclr(unsigned int clear) { __asm__ __volatile__ ("stha %0, [%1] %2" : : "r" (clear), @@ -174,9 +175,9 @@ static inline void cc_set_iclr(unsigned clear) "i" (ASI_M_MXCC)); } -static inline unsigned cc_get_imsk(void) +static inline unsigned int cc_get_imsk(void) { - unsigned mask; + unsigned int mask; __asm__ __volatile__ ("lduha [%1] %2, %0" : "=r" (mask) : @@ -185,7 +186,7 @@ static inline unsigned cc_get_imsk(void) return mask; } -static inline void cc_set_imsk(unsigned mask) +static inline void cc_set_imsk(unsigned int mask) { __asm__ __volatile__ ("stha %0, [%1] %2" : : "r" (mask), @@ -193,9 +194,9 @@ static inline void cc_set_imsk(unsigned mask) "i" (ASI_M_MXCC)); } -static inline unsigned cc_get_imsk_other(int cpuid) +static inline unsigned int cc_get_imsk_other(int cpuid) { - unsigned mask; + unsigned int mask; __asm__ __volatile__ ("lduha [%1] %2, %0" : "=r" (mask) : @@ -204,7 +205,7 @@ static inline unsigned cc_get_imsk_other(int cpuid) return mask; } -static inline void cc_set_imsk_other(int cpuid, unsigned mask) +static inline void cc_set_imsk_other(int cpuid, unsigned int mask) { __asm__ __volatile__ ("stha %0, [%1] %2" : : "r" (mask), @@ -212,7 +213,7 @@ static inline void cc_set_imsk_other(int cpuid, unsigned mask) "i" (ASI_M_CTL)); } -static inline void cc_set_igen(unsigned gen) +static inline void cc_set_igen(unsigned int gen) { __asm__ __volatile__ ("sta %0, [%1] %2" : : "r" (gen), @@ -220,6 +221,6 @@ static inline void cc_set_igen(unsigned gen) "i" (ASI_M_MXCC)); } -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* !(_SPARC_OBIO_H) */ diff --git a/arch/sparc/crypto/opcodes.h b/arch/sparc/include/asm/opcodes.h index 19cbaea6976f..ebfda6eb49b2 100644 --- a/arch/sparc/crypto/opcodes.h +++ b/arch/sparc/include/asm/opcodes.h @@ -1,5 +1,6 @@ -#ifndef _OPCODES_H -#define _OPCODES_H +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_ASM_OPCODES_H +#define _SPARC_ASM_OPCODES_H #define SPARC_CR_OPCODE_PRIORITY 300 @@ -96,4 +97,4 @@ #define MOVXTOD_G7_F62 \ .word 0xbfb02307; -#endif /* _OPCODES_H */ +#endif /* _SPARC_ASM_OPCODES_H */ diff --git a/arch/sparc/include/asm/openprom.h b/arch/sparc/include/asm/openprom.h index 47eaafad15ce..ce68000dffac 100644 --- a/arch/sparc/include/asm/openprom.h +++ b/arch/sparc/include/asm/openprom.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_OPENPROM_H #define __SPARC_OPENPROM_H @@ -10,7 +11,7 @@ /* Empirical constants... */ #define LINUX_OPPROM_MAGIC 0x10010407 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/of.h> /* V0 prom device operations. */ @@ -29,12 +30,12 @@ struct linux_dev_v0_funcs { /* V2 and later prom device operations. */ struct linux_dev_v2_funcs { phandle (*v2_inst2pkg)(int d); /* Convert ihandle to phandle */ - char * (*v2_dumb_mem_alloc)(char *va, unsigned sz); - void (*v2_dumb_mem_free)(char *va, unsigned sz); + char * (*v2_dumb_mem_alloc)(char *va, unsigned int sz); + void (*v2_dumb_mem_free)(char *va, unsigned int sz); /* To map devices into virtual I/O space. */ - char * (*v2_dumb_mmap)(char *virta, int which_io, unsigned paddr, unsigned sz); - void (*v2_dumb_munmap)(char *virta, unsigned size); + char * (*v2_dumb_mmap)(char *virta, int which_io, unsigned int paddr, unsigned int sz); + void (*v2_dumb_munmap)(char *virta, unsigned int size); int (*v2_dev_open)(char *devpath); void (*v2_dev_close)(int d); @@ -50,7 +51,7 @@ struct linux_dev_v2_funcs { struct linux_mlist_v0 { struct linux_mlist_v0 *theres_more; unsigned int start_adr; - unsigned num_bytes; + unsigned int num_bytes; }; struct linux_mem_v0 { @@ -274,6 +275,6 @@ struct linux_prom_pci_intmask { unsigned int interrupt; }; -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #endif /* !(__SPARC_OPENPROM_H) */ diff --git a/arch/sparc/include/asm/oplib.h b/arch/sparc/include/asm/oplib.h index 72e04e13a6b4..df49a7700d73 100644 --- a/arch/sparc/include/asm/oplib.h +++ b/arch/sparc/include/asm/oplib.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_OPLIB_H #define ___ASM_SPARC_OPLIB_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/oplib_32.h b/arch/sparc/include/asm/oplib_32.h index c72f3045820c..d1cf3a27a40d 100644 --- a/arch/sparc/include/asm/oplib_32.h +++ b/arch/sparc/include/asm/oplib_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * oplib.h: Describes the interface and available routines in the * Linux Prom library. @@ -43,28 +44,28 @@ extern struct linux_nodeops *prom_nodeops; /* You must call prom_init() before using any of the library services, * preferably as early as possible. Pass it the romvec pointer. */ -extern void prom_init(struct linux_romvec *rom_ptr); +void prom_init(struct linux_romvec *rom_ptr); /* Boot argument acquisition, returns the boot command line string. */ -extern char *prom_getbootargs(void); +char *prom_getbootargs(void); /* Miscellaneous routines, don't really fit in any category per se. */ /* Reboot the machine with the command line passed. */ -extern void prom_reboot(char *boot_command); +void prom_reboot(char *boot_command); /* Evaluate the forth string passed. */ -extern void prom_feval(char *forth_string); +void prom_feval(char *forth_string); /* Enter the prom, with possibility of continuation with the 'go' * command in newer proms. */ -extern void prom_cmdline(void); +void prom_cmdline(void); /* Enter the prom, with no chance of continuation for the stand-alone * which calls this. */ -extern void __noreturn prom_halt(void); +void __noreturn prom_halt(void); /* Set the PROM 'sync' callback function to the passed function pointer. * When the user gives the 'sync' command at the prom prompt while the @@ -73,37 +74,37 @@ extern void __noreturn prom_halt(void); * XXX The arguments are different on V0 vs. V2->higher proms, grrr! XXX */ typedef void (*sync_func_t)(void); -extern void prom_setsync(sync_func_t func_ptr); +void prom_setsync(sync_func_t func_ptr); /* Acquire the IDPROM of the root node in the prom device tree. This * gets passed a buffer where you would like it stuffed. The return value * is the format type of this idprom or 0xff on error. */ -extern unsigned char prom_get_idprom(char *idp_buffer, int idpbuf_size); +unsigned char prom_get_idprom(char *idp_buffer, int idpbuf_size); /* Get the prom major version. */ -extern int prom_version(void); +int prom_version(void); /* Get the prom plugin revision. */ -extern int prom_getrev(void); +int prom_getrev(void); /* Get the prom firmware revision. */ -extern int prom_getprev(void); +int prom_getprev(void); /* Write a buffer of characters to the console. */ -extern void prom_console_write_buf(const char *buf, int len); +void prom_console_write_buf(const char *buf, int len); /* Prom's internal routines, don't use in kernel/boot code. */ -extern __printf(1, 2) void prom_printf(const char *fmt, ...); -extern void prom_write(const char *buf, unsigned int len); +__printf(1, 2) void prom_printf(const char *fmt, ...); +void prom_write(const char *buf, unsigned int len); /* Multiprocessor operations... */ /* Start the CPU with the given device tree node, context table, and context * at the passed program counter. */ -extern int prom_startcpu(int cpunode, struct linux_prom_registers *context_table, - int context, char *program_counter); +int prom_startcpu(int cpunode, struct linux_prom_registers *context_table, + int context, char *program_counter); /* Initialize the memory lists based upon the prom version. */ void prom_meminit(void); @@ -111,65 +112,65 @@ void prom_meminit(void); /* PROM device tree traversal functions... */ /* Get the child node of the given node, or zero if no child exists. */ -extern phandle prom_getchild(phandle parent_node); +phandle prom_getchild(phandle parent_node); /* Get the next sibling node of the given node, or zero if no further * siblings exist. */ -extern phandle prom_getsibling(phandle node); +phandle prom_getsibling(phandle node); /* Get the length, at the passed node, of the given property type. * Returns -1 on error (ie. no such property at this node). */ -extern int prom_getproplen(phandle thisnode, const char *property); +int prom_getproplen(phandle thisnode, const char *property); /* Fetch the requested property using the given buffer. Returns * the number of bytes the prom put into your buffer or -1 on error. */ -extern int __must_check prom_getproperty(phandle thisnode, const char *property, - char *prop_buffer, int propbuf_size); +int __must_check prom_getproperty(phandle thisnode, const char *property, + char *prop_buffer, int propbuf_size); /* Acquire an integer property. */ -extern int prom_getint(phandle node, char *property); +int prom_getint(phandle node, char *property); /* Acquire an integer property, with a default value. */ -extern int prom_getintdefault(phandle node, char *property, int defval); +int prom_getintdefault(phandle node, char *property, int defval); /* Acquire a boolean property, 0=FALSE 1=TRUE. */ -extern int prom_getbool(phandle node, char *prop); +int prom_getbool(phandle node, char *prop); /* Acquire a string property, null string on error. */ -extern void prom_getstring(phandle node, char *prop, char *buf, int bufsize); +void prom_getstring(phandle node, char *prop, char *buf, int bufsize); /* Search all siblings starting at the passed node for "name" matching * the given string. Returns the node on success, zero on failure. */ -extern phandle prom_searchsiblings(phandle node_start, char *name); +phandle prom_searchsiblings(phandle node_start, char *name); /* Returns the next property after the passed property for the given * node. Returns null string on failure. */ -extern char *prom_nextprop(phandle node, char *prev_property, char *buffer); +char *prom_nextprop(phandle node, char *prev_property, char *buffer); /* Returns phandle of the path specified */ -extern phandle prom_finddevice(char *name); +phandle prom_finddevice(char *name); /* Set the indicated property at the given node with the passed value. * Returns the number of bytes of your value that the prom took. */ -extern int prom_setprop(phandle node, const char *prop_name, char *prop_value, - int value_size); +int prom_setprop(phandle node, const char *prop_name, char *prop_value, + int value_size); -extern phandle prom_inst2pkg(int); +phandle prom_inst2pkg(int); /* Dorking with Bus ranges... */ /* Apply promlib probes OBIO ranges to registers. */ -extern void prom_apply_obio_ranges(struct linux_prom_registers *obioregs, int nregs); +void prom_apply_obio_ranges(struct linux_prom_registers *obioregs, int nregs); /* Apply ranges of any prom node (and optionally parent node as well) to registers. */ -extern void prom_apply_generic_ranges(phandle node, phandle parent, - struct linux_prom_registers *sbusregs, int nregs); +void prom_apply_generic_ranges(phandle node, phandle parent, + struct linux_prom_registers *sbusregs, int nregs); void prom_ranges_init(void); diff --git a/arch/sparc/include/asm/oplib_64.h b/arch/sparc/include/asm/oplib_64.h index a12dbe3b7762..1b86d02a8455 100644 --- a/arch/sparc/include/asm/oplib_64.h +++ b/arch/sparc/include/asm/oplib_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* oplib.h: Describes the interface and available routines in the * Linux Prom library. * @@ -62,100 +63,101 @@ struct linux_mem_p1275 { /* You must call prom_init() before using any of the library services, * preferably as early as possible. Pass it the romvec pointer. */ -extern void prom_init(void *cif_handler, void *cif_stack); +void prom_init(void *cif_handler); +void prom_init_report(void); /* Boot argument acquisition, returns the boot command line string. */ -extern char *prom_getbootargs(void); +char *prom_getbootargs(void); /* Miscellaneous routines, don't really fit in any category per se. */ /* Reboot the machine with the command line passed. */ -extern void prom_reboot(const char *boot_command); +void prom_reboot(const char *boot_command); /* Evaluate the forth string passed. */ -extern void prom_feval(const char *forth_string); +void prom_feval(const char *forth_string); /* Enter the prom, with possibility of continuation with the 'go' * command in newer proms. */ -extern void prom_cmdline(void); +void prom_cmdline(void); /* Enter the prom, with no chance of continuation for the stand-alone * which calls this. */ -extern void prom_halt(void) __attribute__ ((noreturn)); +void prom_halt(void) __attribute__ ((noreturn)); /* Halt and power-off the machine. */ -extern void prom_halt_power_off(void) __attribute__ ((noreturn)); +void prom_halt_power_off(void) __attribute__ ((noreturn)); /* Acquire the IDPROM of the root node in the prom device tree. This * gets passed a buffer where you would like it stuffed. The return value * is the format type of this idprom or 0xff on error. */ -extern unsigned char prom_get_idprom(char *idp_buffer, int idpbuf_size); +unsigned char prom_get_idprom(char *idp_buffer, int idpbuf_size); /* Write a buffer of characters to the console. */ -extern void prom_console_write_buf(const char *buf, int len); +void prom_console_write_buf(const char *buf, int len); /* Prom's internal routines, don't use in kernel/boot code. */ -extern __printf(1, 2) void prom_printf(const char *fmt, ...); -extern void prom_write(const char *buf, unsigned int len); +__printf(1, 2) void prom_printf(const char *fmt, ...); +void prom_write(const char *buf, unsigned int len); /* Multiprocessor operations... */ #ifdef CONFIG_SMP /* Start the CPU with the given device tree node at the passed program * counter with the given arg passed in via register %o0. */ -extern void prom_startcpu(int cpunode, unsigned long pc, unsigned long arg); +void prom_startcpu(int cpunode, unsigned long pc, unsigned long arg); /* Start the CPU with the given cpu ID at the passed program * counter with the given arg passed in via register %o0. */ -extern void prom_startcpu_cpuid(int cpuid, unsigned long pc, unsigned long arg); +void prom_startcpu_cpuid(int cpuid, unsigned long pc, unsigned long arg); /* Stop the CPU with the given cpu ID. */ -extern void prom_stopcpu_cpuid(int cpuid); +void prom_stopcpu_cpuid(int cpuid); /* Stop the current CPU. */ -extern void prom_stopself(void); +void prom_stopself(void); /* Idle the current CPU. */ -extern void prom_idleself(void); +void prom_idleself(void); /* Resume the CPU with the passed device tree node. */ -extern void prom_resumecpu(int cpunode); +void prom_resumecpu(int cpunode); #endif /* Power management interfaces. */ /* Put the current CPU to sleep. */ -extern void prom_sleepself(void); +void prom_sleepself(void); /* Put the entire system to sleep. */ -extern int prom_sleepsystem(void); +int prom_sleepsystem(void); /* Initiate a wakeup event. */ -extern int prom_wakeupsystem(void); +int prom_wakeupsystem(void); /* MMU and memory related OBP interfaces. */ /* Get unique string identifying SIMM at given physical address. */ -extern int prom_getunumber(int syndrome_code, - unsigned long phys_addr, - char *buf, int buflen); +int prom_getunumber(int syndrome_code, + unsigned long phys_addr, + char *buf, int buflen); /* Retain physical memory to the caller across soft resets. */ -extern int prom_retain(const char *name, unsigned long size, - unsigned long align, unsigned long *paddr); +int prom_retain(const char *name, unsigned long size, + unsigned long align, unsigned long *paddr); /* Load explicit I/D TLB entries into the calling processor. */ -extern long prom_itlb_load(unsigned long index, - unsigned long tte_data, - unsigned long vaddr); +long prom_itlb_load(unsigned long index, + unsigned long tte_data, + unsigned long vaddr); -extern long prom_dtlb_load(unsigned long index, - unsigned long tte_data, - unsigned long vaddr); +long prom_dtlb_load(unsigned long index, + unsigned long tte_data, + unsigned long vaddr); /* Map/Unmap client program address ranges. First the format of * the mapping mode argument. @@ -170,81 +172,82 @@ extern long prom_dtlb_load(unsigned long index, #define PROM_MAP_IE 0x0100 /* Invert-Endianness */ #define PROM_MAP_DEFAULT (PROM_MAP_WRITE | PROM_MAP_READ | PROM_MAP_EXEC | PROM_MAP_CACHED) -extern int prom_map(int mode, unsigned long size, - unsigned long vaddr, unsigned long paddr); -extern void prom_unmap(unsigned long size, unsigned long vaddr); +int prom_map(int mode, unsigned long size, + unsigned long vaddr, unsigned long paddr); +void prom_unmap(unsigned long size, unsigned long vaddr); /* PROM device tree traversal functions... */ /* Get the child node of the given node, or zero if no child exists. */ -extern phandle prom_getchild(phandle parent_node); +phandle prom_getchild(phandle parent_node); /* Get the next sibling node of the given node, or zero if no further * siblings exist. */ -extern phandle prom_getsibling(phandle node); +phandle prom_getsibling(phandle node); /* Get the length, at the passed node, of the given property type. * Returns -1 on error (ie. no such property at this node). */ -extern int prom_getproplen(phandle thisnode, const char *property); +int prom_getproplen(phandle thisnode, const char *property); /* Fetch the requested property using the given buffer. Returns * the number of bytes the prom put into your buffer or -1 on error. */ -extern int prom_getproperty(phandle thisnode, const char *property, - char *prop_buffer, int propbuf_size); +int prom_getproperty(phandle thisnode, const char *property, + char *prop_buffer, int propbuf_size); /* Acquire an integer property. */ -extern int prom_getint(phandle node, const char *property); +int prom_getint(phandle node, const char *property); /* Acquire an integer property, with a default value. */ -extern int prom_getintdefault(phandle node, const char *property, int defval); +int prom_getintdefault(phandle node, const char *property, int defval); /* Acquire a boolean property, 0=FALSE 1=TRUE. */ -extern int prom_getbool(phandle node, const char *prop); +int prom_getbool(phandle node, const char *prop); /* Acquire a string property, null string on error. */ -extern void prom_getstring(phandle node, const char *prop, char *buf, - int bufsize); +void prom_getstring(phandle node, const char *prop, char *buf, + int bufsize); /* Does the passed node have the given "name"? YES=1 NO=0 */ -extern int prom_nodematch(phandle thisnode, const char *name); +int prom_nodematch(phandle thisnode, const char *name); /* Search all siblings starting at the passed node for "name" matching * the given string. Returns the node on success, zero on failure. */ -extern phandle prom_searchsiblings(phandle node_start, const char *name); +phandle prom_searchsiblings(phandle node_start, const char *name); /* Return the first property type, as a string, for the given node. * Returns a null string on error. Buffer should be at least 32B long. */ -extern char *prom_firstprop(phandle node, char *buffer); +char *prom_firstprop(phandle node, char *buffer); /* Returns the next property after the passed property for the given * node. Returns null string on failure. Buffer should be at least 32B long. */ -extern char *prom_nextprop(phandle node, const char *prev_property, char *buf); +char *prom_nextprop(phandle node, const char *prev_property, char *buf); /* Returns 1 if the specified node has given property. */ -extern int prom_node_has_property(phandle node, const char *property); +int prom_node_has_property(phandle node, const char *property); /* Returns phandle of the path specified */ -extern phandle prom_finddevice(const char *name); +phandle prom_finddevice(const char *name); /* Set the indicated property at the given node with the passed value. * Returns the number of bytes of your value that the prom took. */ -extern int prom_setprop(phandle node, const char *prop_name, char *prop_value, - int value_size); +int prom_setprop(phandle node, const char *prop_name, char *prop_value, + int value_size); -extern phandle prom_inst2pkg(int); -extern void prom_sun4v_guest_soft_state(void); +phandle prom_inst2pkg(int); +void prom_sun4v_guest_soft_state(void); -extern int prom_ihandle2path(int handle, char *buffer, int bufsize); +int prom_ihandle2path(int handle, char *buffer, int bufsize); /* Client interface level routines. */ -extern void p1275_cmd_direct(unsigned long *); +void prom_cif_init(void *cif_handler); +void p1275_cmd_direct(unsigned long *); #endif /* !(__SPARC64_OPLIB_H) */ diff --git a/arch/sparc/include/asm/page.h b/arch/sparc/include/asm/page.h index f21de0349025..1a00cc0a1893 100644 --- a/arch/sparc/include/asm/page.h +++ b/arch/sparc/include/asm/page.h @@ -1,5 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_PAGE_H #define ___ASM_SPARC_PAGE_H + #if defined(__sparc__) && defined(__arch64__) #include <asm/page_64.h> #else diff --git a/arch/sparc/include/asm/page_32.h b/arch/sparc/include/asm/page_32.h index f82a1f36b655..c1bccbedf567 100644 --- a/arch/sparc/include/asm/page_32.h +++ b/arch/sparc/include/asm/page_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * page.h: Various defines and such for MMU operations on the Sparc for * the Linux kernel. @@ -10,11 +11,9 @@ #include <linux/const.h> -#define PAGE_SHIFT 12 -#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT) -#define PAGE_MASK (~(PAGE_SIZE-1)) +#include <vdso/page.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define clear_page(page) memset((void *)(page), 0, PAGE_SIZE) #define copy_page(to,from) memcpy((void *)(to), (void *)(from), PAGE_SIZE) @@ -53,7 +52,7 @@ extern struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS+1]; */ typedef struct { unsigned long pte; } pte_t; typedef struct { unsigned long iopte; } iopte_t; -typedef struct { unsigned long pmdv[16]; } pmd_t; +typedef struct { unsigned long pmd; } pmd_t; typedef struct { unsigned long pgd; } pgd_t; typedef struct { unsigned long ctxd; } ctxd_t; typedef struct { unsigned long pgprot; } pgprot_t; @@ -61,15 +60,15 @@ typedef struct { unsigned long iopgprot; } iopgprot_t; #define pte_val(x) ((x).pte) #define iopte_val(x) ((x).iopte) -#define pmd_val(x) ((x).pmdv[0]) +#define pmd_val(x) ((x).pmd) #define pgd_val(x) ((x).pgd) #define ctxd_val(x) ((x).ctxd) #define pgprot_val(x) ((x).pgprot) #define iopgprot_val(x) ((x).iopgprot) #define __pte(x) ((pte_t) { (x) } ) +#define __pmd(x) ((pmd_t) { { (x) }, }) #define __iopte(x) ((iopte_t) { (x) } ) -/* #define __pmd(x) ((pmd_t) { (x) } ) */ /* XXX procedure with loop */ #define __pgd(x) ((pgd_t) { (x) } ) #define __ctxd(x) ((ctxd_t) { (x) } ) #define __pgprot(x) ((pgprot_t) { (x) } ) @@ -81,7 +80,7 @@ typedef struct { unsigned long iopgprot; } iopgprot_t; */ typedef unsigned long pte_t; typedef unsigned long iopte_t; -typedef struct { unsigned long pmdv[16]; } pmd_t; +typedef unsigned long pmd_t; typedef unsigned long pgd_t; typedef unsigned long ctxd_t; typedef unsigned long pgprot_t; @@ -89,15 +88,15 @@ typedef unsigned long iopgprot_t; #define pte_val(x) (x) #define iopte_val(x) (x) -#define pmd_val(x) ((x).pmdv[0]) +#define pmd_val(x) (x) #define pgd_val(x) (x) #define ctxd_val(x) (x) #define pgprot_val(x) (x) #define iopgprot_val(x) (x) #define __pte(x) (x) +#define __pmd(x) (x) #define __iopte(x) (x) -/* #define __pmd(x) (x) */ /* XXX later */ #define __pgd(x) (x) #define __ctxd(x) (x) #define __pgprot(x) (x) @@ -105,18 +104,18 @@ typedef unsigned long iopgprot_t; #endif -typedef struct page *pgtable_t; +typedef pte_t *pgtable_t; #define TASK_UNMAPPED_BASE 0x50000000 -#else /* !(__ASSEMBLY__) */ +#else /* !(__ASSEMBLER__) */ #define __pgprot(x) (x) -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #define PAGE_OFFSET 0xf0000000 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ extern unsigned long phys_base; extern unsigned long pfn_base; #endif @@ -129,12 +128,8 @@ extern unsigned long pfn_base; #define ARCH_PFN_OFFSET (pfn_base) #define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) -#define pfn_valid(pfn) (((pfn) >= (pfn_base)) && (((pfn)-(pfn_base)) < max_mapnr)) #define virt_addr_valid(kaddr) ((((unsigned long)(kaddr)-PAGE_OFFSET)>>PAGE_SHIFT) < max_mapnr) -#define VM_DATA_DEFAULT_FLAGS (VM_READ | VM_WRITE | VM_EXEC | \ - VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) - #include <asm-generic/memory_model.h> #include <asm-generic/getorder.h> diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h index e15538899f3d..d764d8a8586b 100644 --- a/arch/sparc/include/asm/page_64.h +++ b/arch/sparc/include/asm/page_64.h @@ -1,12 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_PAGE_H #define _SPARC64_PAGE_H #include <linux/const.h> -#define PAGE_SHIFT 13 - -#define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT) -#define PAGE_MASK (~(PAGE_SIZE-1)) +#include <vdso/page.h> /* Flushing for D-cache alias handling is only needed if * the page size is smaller than 16K. @@ -15,30 +13,44 @@ #define DCACHE_ALIASING_POSSIBLE #endif -#define HPAGE_SHIFT 22 +#define HPAGE_SHIFT 23 +#define REAL_HPAGE_SHIFT 22 +#define HPAGE_16GB_SHIFT 34 +#define HPAGE_2GB_SHIFT 31 +#define HPAGE_256MB_SHIFT 28 +#define HPAGE_64K_SHIFT 16 +#define REAL_HPAGE_SIZE (_AC(1,UL) << REAL_HPAGE_SHIFT) #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) #define HPAGE_SIZE (_AC(1,UL) << HPAGE_SHIFT) #define HPAGE_MASK (~(HPAGE_SIZE - 1UL)) #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) #define HAVE_ARCH_HUGETLB_UNMAPPED_AREA +#define REAL_HPAGE_PER_HPAGE (_AC(1,UL) << (HPAGE_SHIFT - REAL_HPAGE_SHIFT)) +#define HUGE_MAX_HSTATE 5 #endif -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) struct pt_regs; -extern void hugetlb_setup(struct pt_regs *regs); +void hugetlb_setup(struct pt_regs *regs); #endif #define WANT_PAGE_VIRTUAL -extern void _clear_page(void *page); +void _clear_page(void *page); #define clear_page(X) _clear_page((void *)(X)) struct page; -extern void clear_user_page(void *addr, unsigned long vaddr, struct page *page); +void clear_user_page(void *addr, unsigned long vaddr, struct page *page); #define copy_page(X,Y) memcpy((void *)(X), (void *)(Y), PAGE_SIZE) -extern void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *topage); +void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *topage); +#define __HAVE_ARCH_COPY_USER_HIGHPAGE +struct vm_area_struct; +void copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma); +#define __HAVE_ARCH_COPY_HIGHPAGE +void copy_highpage(struct page *to, struct page *from); /* Unlike sparc32, sparc64's parameter passing API is more * sane in that structures which as small enough are passed @@ -53,19 +65,22 @@ extern void copy_user_page(void *to, void *from, unsigned long vaddr, struct pag /* These are used to make use of C type-checking.. */ typedef struct { unsigned long pte; } pte_t; typedef struct { unsigned long iopte; } iopte_t; -typedef struct { unsigned int pmd; } pmd_t; -typedef struct { unsigned int pgd; } pgd_t; +typedef struct { unsigned long pmd; } pmd_t; +typedef struct { unsigned long pud; } pud_t; +typedef struct { unsigned long pgd; } pgd_t; typedef struct { unsigned long pgprot; } pgprot_t; #define pte_val(x) ((x).pte) #define iopte_val(x) ((x).iopte) #define pmd_val(x) ((x).pmd) +#define pud_val(x) ((x).pud) #define pgd_val(x) ((x).pgd) #define pgprot_val(x) ((x).pgprot) #define __pte(x) ((pte_t) { (x) } ) #define __iopte(x) ((iopte_t) { (x) } ) #define __pmd(x) ((pmd_t) { (x) } ) +#define __pud(x) ((pud_t) { (x) } ) #define __pgd(x) ((pgd_t) { (x) } ) #define __pgprot(x) ((pgprot_t) { (x) } ) @@ -73,19 +88,22 @@ typedef struct { unsigned long pgprot; } pgprot_t; /* .. while these make it easier on the compiler */ typedef unsigned long pte_t; typedef unsigned long iopte_t; -typedef unsigned int pmd_t; -typedef unsigned int pgd_t; +typedef unsigned long pmd_t; +typedef unsigned long pud_t; +typedef unsigned long pgd_t; typedef unsigned long pgprot_t; #define pte_val(x) (x) #define iopte_val(x) (x) #define pmd_val(x) (x) +#define pud_val(x) (x) #define pgd_val(x) (x) #define pgprot_val(x) (x) #define __pte(x) (x) #define __iopte(x) (x) #define __pmd(x) (x) +#define __pud(x) (x) #define __pgd(x) (x) #define __pgprot(x) (x) @@ -93,20 +111,35 @@ typedef unsigned long pgprot_t; typedef pte_t *pgtable_t; +extern unsigned long sparc64_va_hole_top; +extern unsigned long sparc64_va_hole_bottom; + +/* The next two defines specify the actual exclusion region we + * enforce, wherein we use a 4GB red zone on each side of the VA hole. + */ +#define VA_EXCLUDE_START (sparc64_va_hole_bottom - (1UL << 32UL)) +#define VA_EXCLUDE_END (sparc64_va_hole_top + (1UL << 32UL)) + #define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \ - (_AC(0x0000000070000000,UL)) : \ - (_AC(0xfffff80000000000,UL) + (1UL << 32UL))) + _AC(0x0000000070000000,UL) : \ + VA_EXCLUDE_END) #include <asm-generic/memory_model.h> -#endif /* !(__ASSEMBLY__) */ +extern unsigned long PAGE_OFFSET; -/* We used to stick this into a hard-coded global register (%g4) - * but that does not make sense anymore. +#endif /* !(__ASSEMBLER__) */ + +/* The maximum number of physical memory address bits we support. The + * largest value we can support is whatever "KPGD_SHIFT + KPTE_BITS" + * evaluates to. */ -#define PAGE_OFFSET _AC(0xFFFFF80000000000,UL) +#define MAX_PHYS_ADDRESS_BITS 53 + +#define ILOG2_4MB 22 +#define ILOG2_256MB 28 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define __pa(x) ((unsigned long)(x) - PAGE_OFFSET) #define __va(x) ((void *)((unsigned long) (x) + PAGE_OFFSET)) @@ -120,10 +153,7 @@ typedef pte_t *pgtable_t; #define virt_to_phys __pa #define phys_to_virt __va -#endif /* !(__ASSEMBLY__) */ - -#define VM_DATA_DEFAULT_FLAGS (VM_READ | VM_WRITE | VM_EXEC | \ - VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) +#endif /* !(__ASSEMBLER__) */ #include <asm-generic/getorder.h> diff --git a/arch/sparc/include/asm/parport.h b/arch/sparc/include/asm/parport.h index c55291e5b83e..e2eed8f97665 100644 --- a/arch/sparc/include/asm/parport.h +++ b/arch/sparc/include/asm/parport.h @@ -1,253 +1,11 @@ -/* parport.h: sparc64 specific parport initialization and dma. - * - * Copyright (C) 1999 Eddie C. Dost (ecd@skynet.be) - */ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_PARPORT_H +#define ___ASM_SPARC_PARPORT_H -#ifndef _ASM_SPARC64_PARPORT_H -#define _ASM_SPARC64_PARPORT_H 1 +#if defined(__sparc__) && defined(__arch64__) +#include <asm/parport_64.h> +#else +#include <asm-generic/parport.h> +#endif +#endif -#include <linux/of_device.h> - -#include <asm/ebus_dma.h> -#include <asm/ns87303.h> -#include <asm/prom.h> - -#define PARPORT_PC_MAX_PORTS PARPORT_MAX - -/* - * While sparc64 doesn't have an ISA DMA API, we provide something that looks - * close enough to make parport_pc happy - */ -#define HAS_DMA - -static DEFINE_SPINLOCK(dma_spin_lock); - -#define claim_dma_lock() \ -({ unsigned long flags; \ - spin_lock_irqsave(&dma_spin_lock, flags); \ - flags; \ -}) - -#define release_dma_lock(__flags) \ - spin_unlock_irqrestore(&dma_spin_lock, __flags); - -static struct sparc_ebus_info { - struct ebus_dma_info info; - unsigned int addr; - unsigned int count; - int lock; - - struct parport *port; -} sparc_ebus_dmas[PARPORT_PC_MAX_PORTS]; - -static DECLARE_BITMAP(dma_slot_map, PARPORT_PC_MAX_PORTS); - -static inline int request_dma(unsigned int dmanr, const char *device_id) -{ - if (dmanr >= PARPORT_PC_MAX_PORTS) - return -EINVAL; - if (xchg(&sparc_ebus_dmas[dmanr].lock, 1) != 0) - return -EBUSY; - return 0; -} - -static inline void free_dma(unsigned int dmanr) -{ - if (dmanr >= PARPORT_PC_MAX_PORTS) { - printk(KERN_WARNING "Trying to free DMA%d\n", dmanr); - return; - } - if (xchg(&sparc_ebus_dmas[dmanr].lock, 0) == 0) { - printk(KERN_WARNING "Trying to free free DMA%d\n", dmanr); - return; - } -} - -static inline void enable_dma(unsigned int dmanr) -{ - ebus_dma_enable(&sparc_ebus_dmas[dmanr].info, 1); - - if (ebus_dma_request(&sparc_ebus_dmas[dmanr].info, - sparc_ebus_dmas[dmanr].addr, - sparc_ebus_dmas[dmanr].count)) - BUG(); -} - -static inline void disable_dma(unsigned int dmanr) -{ - ebus_dma_enable(&sparc_ebus_dmas[dmanr].info, 0); -} - -static inline void clear_dma_ff(unsigned int dmanr) -{ - /* nothing */ -} - -static inline void set_dma_mode(unsigned int dmanr, char mode) -{ - ebus_dma_prepare(&sparc_ebus_dmas[dmanr].info, (mode != DMA_MODE_WRITE)); -} - -static inline void set_dma_addr(unsigned int dmanr, unsigned int addr) -{ - sparc_ebus_dmas[dmanr].addr = addr; -} - -static inline void set_dma_count(unsigned int dmanr, unsigned int count) -{ - sparc_ebus_dmas[dmanr].count = count; -} - -static inline unsigned int get_dma_residue(unsigned int dmanr) -{ - return ebus_dma_residue(&sparc_ebus_dmas[dmanr].info); -} - -static int ecpp_probe(struct platform_device *op) -{ - unsigned long base = op->resource[0].start; - unsigned long config = op->resource[1].start; - unsigned long d_base = op->resource[2].start; - unsigned long d_len; - struct device_node *parent; - struct parport *p; - int slot, err; - - parent = op->dev.of_node->parent; - if (!strcmp(parent->name, "dma")) { - p = parport_pc_probe_port(base, base + 0x400, - op->archdata.irqs[0], PARPORT_DMA_NOFIFO, - op->dev.parent->parent, 0); - if (!p) - return -ENOMEM; - dev_set_drvdata(&op->dev, p); - return 0; - } - - for (slot = 0; slot < PARPORT_PC_MAX_PORTS; slot++) { - if (!test_and_set_bit(slot, dma_slot_map)) - break; - } - err = -ENODEV; - if (slot >= PARPORT_PC_MAX_PORTS) - goto out_err; - - spin_lock_init(&sparc_ebus_dmas[slot].info.lock); - - d_len = (op->resource[2].end - d_base) + 1UL; - sparc_ebus_dmas[slot].info.regs = - of_ioremap(&op->resource[2], 0, d_len, "ECPP DMA"); - - if (!sparc_ebus_dmas[slot].info.regs) - goto out_clear_map; - - sparc_ebus_dmas[slot].info.flags = 0; - sparc_ebus_dmas[slot].info.callback = NULL; - sparc_ebus_dmas[slot].info.client_cookie = NULL; - sparc_ebus_dmas[slot].info.irq = 0xdeadbeef; - strcpy(sparc_ebus_dmas[slot].info.name, "parport"); - if (ebus_dma_register(&sparc_ebus_dmas[slot].info)) - goto out_unmap_regs; - - ebus_dma_irq_enable(&sparc_ebus_dmas[slot].info, 1); - - /* Configure IRQ to Push Pull, Level Low */ - /* Enable ECP, set bit 2 of the CTR first */ - outb(0x04, base + 0x02); - ns87303_modify(config, PCR, - PCR_EPP_ENABLE | - PCR_IRQ_ODRAIN, - PCR_ECP_ENABLE | - PCR_ECP_CLK_ENA | - PCR_IRQ_POLAR); - - /* CTR bit 5 controls direction of port */ - ns87303_modify(config, PTR, - 0, PTR_LPT_REG_DIR); - - p = parport_pc_probe_port(base, base + 0x400, - op->archdata.irqs[0], - slot, - op->dev.parent, - 0); - err = -ENOMEM; - if (!p) - goto out_disable_irq; - - dev_set_drvdata(&op->dev, p); - - return 0; - -out_disable_irq: - ebus_dma_irq_enable(&sparc_ebus_dmas[slot].info, 0); - ebus_dma_unregister(&sparc_ebus_dmas[slot].info); - -out_unmap_regs: - of_iounmap(&op->resource[2], sparc_ebus_dmas[slot].info.regs, d_len); - -out_clear_map: - clear_bit(slot, dma_slot_map); - -out_err: - return err; -} - -static int ecpp_remove(struct platform_device *op) -{ - struct parport *p = dev_get_drvdata(&op->dev); - int slot = p->dma; - - parport_pc_unregister_port(p); - - if (slot != PARPORT_DMA_NOFIFO) { - unsigned long d_base = op->resource[2].start; - unsigned long d_len; - - d_len = (op->resource[2].end - d_base) + 1UL; - - ebus_dma_irq_enable(&sparc_ebus_dmas[slot].info, 0); - ebus_dma_unregister(&sparc_ebus_dmas[slot].info); - of_iounmap(&op->resource[2], - sparc_ebus_dmas[slot].info.regs, - d_len); - clear_bit(slot, dma_slot_map); - } - - return 0; -} - -static const struct of_device_id ecpp_match[] = { - { - .name = "ecpp", - }, - { - .name = "parallel", - .compatible = "ecpp", - }, - { - .name = "parallel", - .compatible = "ns87317-ecpp", - }, - { - .name = "parallel", - .compatible = "pnpALI,1533,3", - }, - {}, -}; - -static struct platform_driver ecpp_driver = { - .driver = { - .name = "ecpp", - .owner = THIS_MODULE, - .of_match_table = ecpp_match, - }, - .probe = ecpp_probe, - .remove = ecpp_remove, -}; - -static int parport_pc_find_nonpci_ports(int autoirq, int autodma) -{ - return platform_driver_register(&ecpp_driver); -} - -#endif /* !(_ASM_SPARC64_PARPORT_H */ diff --git a/arch/sparc/include/asm/parport_64.h b/arch/sparc/include/asm/parport_64.h new file mode 100644 index 000000000000..78f14d6620bf --- /dev/null +++ b/arch/sparc/include/asm/parport_64.h @@ -0,0 +1,255 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* parport.h: sparc64 specific parport initialization and dma. + * + * Copyright (C) 1999 Eddie C. Dost (ecd@skynet.be) + */ + +#ifndef _ASM_SPARC64_PARPORT_H +#define _ASM_SPARC64_PARPORT_H 1 + +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/string.h> + +#include <asm/ebus_dma.h> +#include <asm/ns87303.h> +#include <asm/prom.h> + +#define PARPORT_PC_MAX_PORTS PARPORT_MAX + +/* + * While sparc64 doesn't have an ISA DMA API, we provide something that looks + * close enough to make parport_pc happy + */ +#define HAS_DMA + +#ifdef CONFIG_PARPORT_PC_FIFO +static DEFINE_SPINLOCK(dma_spin_lock); + +#define claim_dma_lock() \ +({ unsigned long flags; \ + spin_lock_irqsave(&dma_spin_lock, flags); \ + flags; \ +}) + +#define release_dma_lock(__flags) \ + spin_unlock_irqrestore(&dma_spin_lock, __flags); +#endif + +static struct sparc_ebus_info { + struct ebus_dma_info info; + unsigned int addr; + unsigned int count; + int lock; + + struct parport *port; +} sparc_ebus_dmas[PARPORT_PC_MAX_PORTS]; + +static DECLARE_BITMAP(dma_slot_map, PARPORT_PC_MAX_PORTS); + +static inline int request_dma(unsigned int dmanr, const char *device_id) +{ + if (dmanr >= PARPORT_PC_MAX_PORTS) + return -EINVAL; + if (xchg(&sparc_ebus_dmas[dmanr].lock, 1) != 0) + return -EBUSY; + return 0; +} + +static inline void free_dma(unsigned int dmanr) +{ + if (dmanr >= PARPORT_PC_MAX_PORTS) { + printk(KERN_WARNING "Trying to free DMA%d\n", dmanr); + return; + } + if (xchg(&sparc_ebus_dmas[dmanr].lock, 0) == 0) { + printk(KERN_WARNING "Trying to free free DMA%d\n", dmanr); + return; + } +} + +static inline void enable_dma(unsigned int dmanr) +{ + ebus_dma_enable(&sparc_ebus_dmas[dmanr].info, 1); + + if (ebus_dma_request(&sparc_ebus_dmas[dmanr].info, + sparc_ebus_dmas[dmanr].addr, + sparc_ebus_dmas[dmanr].count)) + BUG(); +} + +static inline void disable_dma(unsigned int dmanr) +{ + ebus_dma_enable(&sparc_ebus_dmas[dmanr].info, 0); +} + +static inline void clear_dma_ff(unsigned int dmanr) +{ + /* nothing */ +} + +static inline void set_dma_mode(unsigned int dmanr, char mode) +{ + ebus_dma_prepare(&sparc_ebus_dmas[dmanr].info, (mode != DMA_MODE_WRITE)); +} + +static inline void set_dma_addr(unsigned int dmanr, unsigned int addr) +{ + sparc_ebus_dmas[dmanr].addr = addr; +} + +static inline void set_dma_count(unsigned int dmanr, unsigned int count) +{ + sparc_ebus_dmas[dmanr].count = count; +} + +static inline unsigned int get_dma_residue(unsigned int dmanr) +{ + return ebus_dma_residue(&sparc_ebus_dmas[dmanr].info); +} + +static int ecpp_probe(struct platform_device *op) +{ + unsigned long base = op->resource[0].start; + unsigned long config = op->resource[1].start; + unsigned long d_base = op->resource[2].start; + unsigned long d_len; + struct device_node *parent; + struct parport *p; + int slot, err; + + parent = op->dev.of_node->parent; + if (of_node_name_eq(parent, "dma")) { + p = parport_pc_probe_port(base, base + 0x400, + op->archdata.irqs[0], PARPORT_DMA_NOFIFO, + op->dev.parent->parent, 0); + if (!p) + return -ENOMEM; + dev_set_drvdata(&op->dev, p); + return 0; + } + + for (slot = 0; slot < PARPORT_PC_MAX_PORTS; slot++) { + if (!test_and_set_bit(slot, dma_slot_map)) + break; + } + err = -ENODEV; + if (slot >= PARPORT_PC_MAX_PORTS) + goto out_err; + + spin_lock_init(&sparc_ebus_dmas[slot].info.lock); + + d_len = (op->resource[2].end - d_base) + 1UL; + sparc_ebus_dmas[slot].info.regs = + of_ioremap(&op->resource[2], 0, d_len, "ECPP DMA"); + + if (!sparc_ebus_dmas[slot].info.regs) + goto out_clear_map; + + sparc_ebus_dmas[slot].info.flags = 0; + sparc_ebus_dmas[slot].info.callback = NULL; + sparc_ebus_dmas[slot].info.client_cookie = NULL; + sparc_ebus_dmas[slot].info.irq = 0xdeadbeef; + strscpy(sparc_ebus_dmas[slot].info.name, "parport"); + if (ebus_dma_register(&sparc_ebus_dmas[slot].info)) + goto out_unmap_regs; + + ebus_dma_irq_enable(&sparc_ebus_dmas[slot].info, 1); + + /* Configure IRQ to Push Pull, Level Low */ + /* Enable ECP, set bit 2 of the CTR first */ + outb(0x04, base + 0x02); + ns87303_modify(config, PCR, + PCR_EPP_ENABLE | + PCR_IRQ_ODRAIN, + PCR_ECP_ENABLE | + PCR_ECP_CLK_ENA | + PCR_IRQ_POLAR); + + /* CTR bit 5 controls direction of port */ + ns87303_modify(config, PTR, + 0, PTR_LPT_REG_DIR); + + p = parport_pc_probe_port(base, base + 0x400, + op->archdata.irqs[0], + slot, + op->dev.parent, + 0); + err = -ENOMEM; + if (!p) + goto out_disable_irq; + + dev_set_drvdata(&op->dev, p); + + return 0; + +out_disable_irq: + ebus_dma_irq_enable(&sparc_ebus_dmas[slot].info, 0); + ebus_dma_unregister(&sparc_ebus_dmas[slot].info); + +out_unmap_regs: + of_iounmap(&op->resource[2], sparc_ebus_dmas[slot].info.regs, d_len); + +out_clear_map: + clear_bit(slot, dma_slot_map); + +out_err: + return err; +} + +static void ecpp_remove(struct platform_device *op) +{ + struct parport *p = dev_get_drvdata(&op->dev); + int slot = p->dma; + + parport_pc_unregister_port(p); + + if (slot != PARPORT_DMA_NOFIFO) { + unsigned long d_base = op->resource[2].start; + unsigned long d_len; + + d_len = (op->resource[2].end - d_base) + 1UL; + + ebus_dma_irq_enable(&sparc_ebus_dmas[slot].info, 0); + ebus_dma_unregister(&sparc_ebus_dmas[slot].info); + of_iounmap(&op->resource[2], + sparc_ebus_dmas[slot].info.regs, + d_len); + clear_bit(slot, dma_slot_map); + } +} + +static const struct of_device_id ecpp_match[] = { + { + .name = "ecpp", + }, + { + .name = "parallel", + .compatible = "ecpp", + }, + { + .name = "parallel", + .compatible = "ns87317-ecpp", + }, + { + .name = "parallel", + .compatible = "pnpALI,1533,3", + }, + {}, +}; + +static struct platform_driver ecpp_driver = { + .driver = { + .name = "ecpp", + .of_match_table = ecpp_match, + }, + .probe = ecpp_probe, + .remove = ecpp_remove, +}; + +static int parport_pc_find_nonpci_ports(int autoirq, int autodma) +{ + return platform_driver_register(&ecpp_driver); +} + +#endif /* !(_ASM_SPARC64_PARPORT_H */ diff --git a/arch/sparc/include/asm/pbm.h b/arch/sparc/include/asm/pbm.h index 458a4916d14d..0c86261721e7 100644 --- a/arch/sparc/include/asm/pbm.h +++ b/arch/sparc/include/asm/pbm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * * pbm.h: PCI bus module pseudo driver software state diff --git a/arch/sparc/include/asm/pci.h b/arch/sparc/include/asm/pci.h index d9c031f9910f..1419aa84df71 100644 --- a/arch/sparc/include/asm/pci.h +++ b/arch/sparc/include/asm/pci.h @@ -1,11 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_PCI_H #define ___ASM_SPARC_PCI_H -#if defined(__sparc__) && defined(__arch64__) -#include <asm/pci_64.h> -#else -#include <asm/pci_32.h> -#endif -#include <asm-generic/pci-dma-compat.h> -#endif +/* Can be used to override the logic in pci_scan_bus for skipping + * already-configured bus numbers - to be used for buggy BIOSes + * or architectures with incomplete PCI setup by the loader. + */ +#define pcibios_assign_all_busses() 0 + +#define PCIBIOS_MIN_IO 0UL +#define PCIBIOS_MIN_MEM 0UL + +#define PCI_IRQ_NONE 0xffffffff + + +#ifdef CONFIG_SPARC64 + +/* PCI IOMMU mapping bypass support. */ + +/* PCI 64-bit addressing works for all slots on all controller + * types on sparc64. However, it requires that the device + * can drive enough of the 64 bits. + */ +#define PCI64_REQUIRED_MASK (~(u64)0) +#define PCI64_ADDR_BASE 0xfffc000000000000UL + +/* Return the index of the PCI controller for device PDEV. */ +int pci_domain_nr(struct pci_bus *bus); +static inline int pci_proc_domain(struct pci_bus *bus) +{ + return 1; +} + +/* Platform support for /proc/bus/pci/X/Y mmap()s. */ +#define HAVE_PCI_MMAP +#define arch_can_pci_mmap_io() 1 +#define HAVE_ARCH_PCI_GET_UNMAPPED_AREA +#define ARCH_GENERIC_PCI_MMAP_RESOURCE +#define get_pci_unmapped_area get_fb_unmapped_area +#endif /* CONFIG_SPARC64 */ + +#endif /* ___ASM_SPARC_PCI_H */ diff --git a/arch/sparc/include/asm/pci_32.h b/arch/sparc/include/asm/pci_32.h deleted file mode 100644 index dc503297481f..000000000000 --- a/arch/sparc/include/asm/pci_32.h +++ /dev/null @@ -1,61 +0,0 @@ -#ifndef __SPARC_PCI_H -#define __SPARC_PCI_H - -#ifdef __KERNEL__ - -#include <linux/dma-mapping.h> - -/* Can be used to override the logic in pci_scan_bus for skipping - * already-configured bus numbers - to be used for buggy BIOSes - * or architectures with incomplete PCI setup by the loader. - */ -#define pcibios_assign_all_busses() 0 - -#define PCIBIOS_MIN_IO 0UL -#define PCIBIOS_MIN_MEM 0UL - -#define PCI_IRQ_NONE 0xffffffff - -static inline void pcibios_penalize_isa_irq(int irq, int active) -{ - /* We don't do dynamic PCI IRQ allocation */ -} - -/* Dynamic DMA mapping stuff. - */ -#define PCI_DMA_BUS_IS_PHYS (0) - -struct pci_dev; - -#ifdef CONFIG_PCI -static inline void pci_dma_burst_advice(struct pci_dev *pdev, - enum pci_dma_burst_strategy *strat, - unsigned long *strategy_parameter) -{ - *strat = PCI_DMA_BURST_INFINITY; - *strategy_parameter = ~0UL; -} -#endif - -#endif /* __KERNEL__ */ - -#ifndef CONFIG_LEON_PCI -/* generic pci stuff */ -#include <asm-generic/pci.h> -#else -/* - * On LEON PCI Memory space is mapped 1:1 with physical address space. - * - * I/O space is located at low 64Kbytes in PCI I/O space. The I/O addresses - * are converted into CPU addresses to virtual addresses that are mapped with - * MMU to the PCI Host PCI I/O space window which are translated to the low - * 64Kbytes by the Host controller. - */ - -static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel) -{ - return PCI_IRQ_NONE; -} -#endif - -#endif /* __SPARC_PCI_H */ diff --git a/arch/sparc/include/asm/pci_64.h b/arch/sparc/include/asm/pci_64.h deleted file mode 100644 index 1633b718d3bc..000000000000 --- a/arch/sparc/include/asm/pci_64.h +++ /dev/null @@ -1,87 +0,0 @@ -#ifndef __SPARC64_PCI_H -#define __SPARC64_PCI_H - -#ifdef __KERNEL__ - -#include <linux/dma-mapping.h> - -/* Can be used to override the logic in pci_scan_bus for skipping - * already-configured bus numbers - to be used for buggy BIOSes - * or architectures with incomplete PCI setup by the loader. - */ -#define pcibios_assign_all_busses() 0 - -#define PCIBIOS_MIN_IO 0UL -#define PCIBIOS_MIN_MEM 0UL - -#define PCI_IRQ_NONE 0xffffffff - -static inline void pcibios_penalize_isa_irq(int irq, int active) -{ - /* We don't do dynamic PCI IRQ allocation */ -} - -/* The PCI address space does not equal the physical memory - * address space. The networking and block device layers use - * this boolean for bounce buffer decisions. - */ -#define PCI_DMA_BUS_IS_PHYS (0) - -/* PCI IOMMU mapping bypass support. */ - -/* PCI 64-bit addressing works for all slots on all controller - * types on sparc64. However, it requires that the device - * can drive enough of the 64 bits. - */ -#define PCI64_REQUIRED_MASK (~(u64)0) -#define PCI64_ADDR_BASE 0xfffc000000000000UL - -#ifdef CONFIG_PCI -static inline void pci_dma_burst_advice(struct pci_dev *pdev, - enum pci_dma_burst_strategy *strat, - unsigned long *strategy_parameter) -{ - unsigned long cacheline_size; - u8 byte; - - pci_read_config_byte(pdev, PCI_CACHE_LINE_SIZE, &byte); - if (byte == 0) - cacheline_size = 1024; - else - cacheline_size = (int) byte * 4; - - *strat = PCI_DMA_BURST_BOUNDARY; - *strategy_parameter = cacheline_size; -} -#endif - -/* Return the index of the PCI controller for device PDEV. */ - -extern int pci_domain_nr(struct pci_bus *bus); -static inline int pci_proc_domain(struct pci_bus *bus) -{ - return 1; -} - -/* Platform support for /proc/bus/pci/X/Y mmap()s. */ - -#define HAVE_PCI_MMAP -#define HAVE_ARCH_PCI_GET_UNMAPPED_AREA -#define get_pci_unmapped_area get_fb_unmapped_area - -extern int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, - enum pci_mmap_state mmap_state, - int write_combine); - -static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel) -{ - return PCI_IRQ_NONE; -} - -#define HAVE_ARCH_PCI_RESOURCE_TO_USER -extern void pci_resource_to_user(const struct pci_dev *dev, int bar, - const struct resource *rsrc, - resource_size_t *start, resource_size_t *end); -#endif /* __KERNEL__ */ - -#endif /* __SPARC64_PCI_H */ diff --git a/arch/sparc/include/asm/pcic.h b/arch/sparc/include/asm/pcic.h index 6676cbcc8b6a..fb5ed6a59535 100644 --- a/arch/sparc/include/asm/pcic.h +++ b/arch/sparc/include/asm/pcic.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * pcic.h: JavaEngine 1 specific PCI definitions. * @@ -7,7 +8,7 @@ #ifndef __SPARC_PCIC_H #define __SPARC_PCIC_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> #include <linux/smp.h> @@ -30,10 +31,10 @@ struct linux_pcic { }; #ifdef CONFIG_PCIC_PCI -extern int pcic_present(void); -extern int pcic_probe(void); -extern void pci_time_init(void); -extern void sun4m_pci_init_IRQ(void); +int pcic_present(void); +int pcic_probe(void); +void pci_time_init(void); +void sun4m_pci_init_IRQ(void); #else static inline int pcic_present(void) { return 0; } static inline int pcic_probe(void) { return 0; } diff --git a/arch/sparc/include/asm/pcr.h b/arch/sparc/include/asm/pcr.h index 942bb17f60cd..da834ffbe75d 100644 --- a/arch/sparc/include/asm/pcr.h +++ b/arch/sparc/include/asm/pcr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __PCR_H #define __PCR_H @@ -12,8 +13,8 @@ struct pcr_ops { }; extern const struct pcr_ops *pcr_ops; -extern void deferred_pcr_work_irq(int irq, struct pt_regs *regs); -extern void schedule_deferred_pcr_work(void); +void deferred_pcr_work_irq(int irq, struct pt_regs *regs); +void schedule_deferred_pcr_work(void); #define PCR_PIC_PRIV 0x00000001 /* PIC access is privileged */ #define PCR_STRACE 0x00000002 /* Trace supervisor events */ @@ -45,6 +46,6 @@ extern void schedule_deferred_pcr_work(void); #define PCR_N4_PICNHT 0x00020000 /* PIC non-hypervisor trap */ #define PCR_N4_NTC 0x00040000 /* Next-To-Commit wrap */ -extern int pcr_arch_init(void); +int pcr_arch_init(void); #endif /* __PCR_H */ diff --git a/arch/sparc/include/asm/percpu.h b/arch/sparc/include/asm/percpu.h index bfb1d19ff1bf..0cd704ef5ff9 100644 --- a/arch/sparc/include/asm/percpu.h +++ b/arch/sparc/include/asm/percpu.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_PERCPU_H #define ___ASM_SPARC_PERCPU_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/percpu_32.h b/arch/sparc/include/asm/percpu_32.h index 06066a7aaec3..ee6c7c1934f6 100644 --- a/arch/sparc/include/asm/percpu_32.h +++ b/arch/sparc/include/asm/percpu_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ARCH_SPARC_PERCPU__ #define __ARCH_SPARC_PERCPU__ diff --git a/arch/sparc/include/asm/percpu_64.h b/arch/sparc/include/asm/percpu_64.h index 007aafb4ae97..a8786a4b90b6 100644 --- a/arch/sparc/include/asm/percpu_64.h +++ b/arch/sparc/include/asm/percpu_64.h @@ -1,9 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ARCH_SPARC64_PERCPU__ #define __ARCH_SPARC64_PERCPU__ #include <linux/compiler.h> +#ifndef BUILD_VDSO register unsigned long __local_per_cpu_offset asm("g5"); +#endif #ifdef CONFIG_SMP diff --git a/arch/sparc/include/asm/perf_event.h b/arch/sparc/include/asm/perf_event.h index 4d3dbe3703e9..c2aec0c7f4f5 100644 --- a/arch/sparc/include/asm/perf_event.h +++ b/arch/sparc/include/asm/perf_event.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASM_SPARC_PERF_EVENT_H #define __ASM_SPARC_PERF_EVENT_H diff --git a/arch/sparc/include/asm/pgalloc.h b/arch/sparc/include/asm/pgalloc.h index b6db1f7cdcab..9ea0b37586c9 100644 --- a/arch/sparc/include/asm/pgalloc.h +++ b/arch/sparc/include/asm/pgalloc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_PGALLOC_H #define ___ASM_SPARC_PGALLOC_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/pgalloc_32.h b/arch/sparc/include/asm/pgalloc_32.h index 9b1c36de0f18..4f73e87b22a3 100644 --- a/arch/sparc/include/asm/pgalloc_32.h +++ b/arch/sparc/include/asm/pgalloc_32.h @@ -1,11 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_PGALLOC_H #define _SPARC_PGALLOC_H #include <linux/kernel.h> #include <linux/sched.h> +#include <linux/pgtable.h> #include <asm/pgtsrmmu.h> -#include <asm/pgtable.h> #include <asm/vaddrs.h> #include <asm/page.h> @@ -14,7 +15,7 @@ struct page; void *srmmu_get_nocache(int size, int align); void srmmu_free_nocache(void *addr, int size); -#define check_pgt_cache() do { } while (0) +extern struct resource sparc_iomap; pgd_t *get_pgd_fast(void); static inline void free_pgd_fast(pgd_t *pgd) @@ -25,14 +26,14 @@ static inline void free_pgd_fast(pgd_t *pgd) #define pgd_free(mm, pgd) free_pgd_fast(pgd) #define pgd_alloc(mm) get_pgd_fast() -static inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp) +static inline void pud_set(pud_t * pudp, pmd_t * pmdp) { - unsigned long pa = __nocache_pa((unsigned long)pmdp); + unsigned long pa = __nocache_pa(pmdp); - set_pte((pte_t *)pgdp, (SRMMU_ET_PTD | (pa >> 4))); + set_pte((pte_t *)pudp, __pte((SRMMU_ET_PTD | (pa >> 4)))); } -#define pgd_populate(MM, PGD, PMD) pgd_set(PGD, PMD) +#define pud_populate(MM, PGD, PMD) pud_set(PGD, PMD) static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) @@ -49,24 +50,23 @@ static inline void free_pmd_fast(pmd_t * pmd) #define pmd_free(mm, pmd) free_pmd_fast(pmd) #define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) -void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep); -#define pmd_pgtable(pmd) pmd_page(pmd) +#define pmd_populate(mm, pmd, pte) pmd_set(pmd, pte) void pmd_set(pmd_t *pmdp, pte_t *ptep); -#define pmd_populate_kernel(MM, PMD, PTE) pmd_set(PMD, PTE) +#define pmd_populate_kernel pmd_populate -pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address); +pgtable_t pte_alloc_one(struct mm_struct *mm); -static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, - unsigned long address) +static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) { - return srmmu_get_nocache(PTE_SIZE, PTE_SIZE); + return srmmu_get_nocache(SRMMU_PTE_TABLE_SIZE, + SRMMU_PTE_TABLE_SIZE); } static inline void free_pte_fast(pte_t *pte) { - srmmu_free_nocache(pte, PTE_SIZE); + srmmu_free_nocache(pte, SRMMU_PTE_TABLE_SIZE); } #define pte_free_kernel(mm, pte) free_pte_fast(pte) diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h index bcfe063bce23..caa7632be4c2 100644 --- a/arch/sparc/include/asm/pgalloc_64.h +++ b/arch/sparc/include/asm/pgalloc_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_PGALLOC_H #define _SPARC64_PGALLOC_H @@ -15,6 +16,13 @@ extern struct kmem_cache *pgtable_cache; +static inline void __p4d_populate(p4d_t *p4d, pud_t *pud) +{ + p4d_set(p4d, pud); +} + +#define p4d_populate(MM, P4D, PUD) __p4d_populate(P4D, PUD) + static inline pgd_t *pgd_alloc(struct mm_struct *mm) { return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); @@ -25,12 +33,26 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) kmem_cache_free(pgtable_cache, pgd); } -#define pud_populate(MM, PUD, PMD) pud_set(PUD, PMD) +static inline void __pud_populate(pud_t *pud, pmd_t *pmd) +{ + pud_set(pud, pmd); +} + +#define pud_populate(MM, PUD, PMD) __pud_populate(PUD, PMD) + +static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); +} + +static inline void pud_free(struct mm_struct *mm, pud_t *pud) +{ + kmem_cache_free(pgtable_cache, pud); +} static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { - return kmem_cache_alloc(pgtable_cache, - GFP_KERNEL|__GFP_REPEAT); + return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); } static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) @@ -38,25 +60,24 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) kmem_cache_free(pgtable_cache, pmd); } -extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, - unsigned long address); -extern pgtable_t pte_alloc_one(struct mm_struct *mm, - unsigned long address); -extern void pte_free_kernel(struct mm_struct *mm, pte_t *pte); -extern void pte_free(struct mm_struct *mm, pgtable_t ptepage); +pte_t *pte_alloc_one_kernel(struct mm_struct *mm); +pgtable_t pte_alloc_one(struct mm_struct *mm); +void pte_free_kernel(struct mm_struct *mm, pte_t *pte); +void pte_free(struct mm_struct *mm, pgtable_t ptepage); + +/* arch use pte_free_defer() implementation in arch/sparc/mm/init_64.c */ +#define pte_free_defer pte_free_defer +void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable); #define pmd_populate_kernel(MM, PMD, PTE) pmd_set(MM, PMD, PTE) #define pmd_populate(MM, PMD, PTE) pmd_set(MM, PMD, PTE) -#define pmd_pgtable(PMD) ((pte_t *)__pmd_page(PMD)) -#define check_pgt_cache() do { } while (0) - -extern void pgtable_free(void *table, bool is_page); +void pgtable_free(void *table, bool is_page); #ifdef CONFIG_SMP struct mmu_gather; -extern void tlb_remove_table(struct mmu_gather *, void *); +void tlb_remove_table(struct mmu_gather *, void *); static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) { @@ -91,4 +112,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte, #define __pmd_free_tlb(tlb, pmd, addr) \ pgtable_free_tlb(tlb, pmd, false) +#define __pud_free_tlb(tlb, pud, addr) \ + pgtable_free_tlb(tlb, pud, false) + #endif /* _SPARC64_PGALLOC_H */ diff --git a/arch/sparc/include/asm/pgtable.h b/arch/sparc/include/asm/pgtable.h index 59ba6f620732..e48891170f84 100644 --- a/arch/sparc/include/asm/pgtable.h +++ b/arch/sparc/include/asm/pgtable.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_PGTABLE_H #define ___ASM_SPARC_PGTABLE_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h index 502f632f6cc7..a9f802d1dd64 100644 --- a/arch/sparc/include/asm/pgtable_32.h +++ b/arch/sparc/include/asm/pgtable_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_PGTABLE_H #define _SPARC_PGTABLE_H @@ -10,11 +11,21 @@ #include <linux/const.h> -#ifndef __ASSEMBLY__ -#include <asm-generic/4level-fixup.h> +#define PMD_SHIFT 18 +#define PMD_SIZE (1UL << PMD_SHIFT) +#define PMD_MASK (~(PMD_SIZE-1)) +#define PMD_ALIGN(__addr) (((__addr) + ~PMD_MASK) & PMD_MASK) + +#define PGDIR_SHIFT 24 +#define PGDIR_SIZE (1UL << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE-1)) +#define PGDIR_ALIGN(__addr) (((__addr) + ~PGDIR_MASK) & PGDIR_MASK) + +#ifndef __ASSEMBLER__ +#include <asm-generic/pgtable-nopud.h> #include <linux/spinlock.h> -#include <linux/swap.h> +#include <linux/mm_types.h> #include <asm/types.h> #include <asm/pgtsrmmu.h> #include <asm/vaddrs.h> @@ -25,25 +36,18 @@ struct vm_area_struct; struct page; -extern void load_mmu(void); -extern unsigned long calc_highpages(void); +void load_mmu(void); +unsigned long calc_highpages(void); +unsigned long __init bootmem_init(unsigned long *pages_avail); #define pte_ERROR(e) __builtin_trap() #define pmd_ERROR(e) __builtin_trap() #define pgd_ERROR(e) __builtin_trap() -#define PMD_SHIFT 22 -#define PMD_SIZE (1UL << PMD_SHIFT) -#define PMD_MASK (~(PMD_SIZE-1)) -#define PMD_ALIGN(__addr) (((__addr) + ~PMD_MASK) & PMD_MASK) -#define PGDIR_SHIFT SRMMU_PGDIR_SHIFT -#define PGDIR_SIZE SRMMU_PGDIR_SIZE -#define PGDIR_MASK SRMMU_PGDIR_MASK -#define PTRS_PER_PTE 1024 -#define PTRS_PER_PMD SRMMU_PTRS_PER_PMD -#define PTRS_PER_PGD SRMMU_PTRS_PER_PGD -#define USER_PTRS_PER_PGD PAGE_OFFSET / SRMMU_PGDIR_SIZE -#define FIRST_USER_ADDRESS 0 +#define PTRS_PER_PTE 64 +#define PTRS_PER_PMD 64 +#define PTRS_PER_PGD 256 +#define USER_PTRS_PER_PGD PAGE_OFFSET / PGDIR_SIZE #define PTE_SIZE (PTRS_PER_PTE*4) #define PAGE_NONE SRMMU_PAGE_NONE @@ -56,29 +60,10 @@ extern unsigned long calc_highpages(void); * srmmu.c will assign the real one (which is dynamically sized) */ #define swapper_pg_dir NULL -extern void paging_init(void); +void paging_init(void); extern unsigned long ptr_in_current_pgd; -/* xwr */ -#define __P000 PAGE_NONE -#define __P001 PAGE_READONLY -#define __P010 PAGE_COPY -#define __P011 PAGE_COPY -#define __P100 PAGE_READONLY -#define __P101 PAGE_READONLY -#define __P110 PAGE_COPY -#define __P111 PAGE_COPY - -#define __S000 PAGE_NONE -#define __S001 PAGE_READONLY -#define __S010 PAGE_SHARED -#define __S011 PAGE_SHARED -#define __S100 PAGE_READONLY -#define __S101 PAGE_READONLY -#define __S110 PAGE_SHARED -#define __S111 PAGE_SHARED - /* First physical page can be anywhere, the following is needed so that * va-->pa and vice versa conversions work properly without performance * hit for all __pa()/__va() operations. @@ -90,9 +75,9 @@ extern unsigned long pfn_base; * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. */ -extern unsigned long empty_zero_page; +extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; -#define ZERO_PAGE(vaddr) (virt_to_page(&empty_zero_page)) +#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) /* * In general all page table modifications should use the V8 atomic @@ -101,7 +86,8 @@ extern unsigned long empty_zero_page; */ static inline unsigned long srmmu_swap(unsigned long *addr, unsigned long value) { - __asm__ __volatile__("swap [%2], %0" : "=&r" (value) : "0" (value), "r" (addr)); + __asm__ __volatile__("swap [%2], %0" : + "=&r" (value) : "0" (value), "r" (addr) : "memory"); return value; } @@ -115,27 +101,47 @@ static inline void set_pte(pte_t *ptep, pte_t pteval) srmmu_swap((unsigned long *)ptep, pte_val(pteval)); } -#define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) - static inline int srmmu_device_memory(unsigned long x) { return ((x & 0xF0000000) != 0); } +static inline unsigned long pmd_pfn(pmd_t pmd) +{ + return (pmd_val(pmd) & SRMMU_PTD_PMASK) >> (PAGE_SHIFT-4); +} + static inline struct page *pmd_page(pmd_t pmd) { if (srmmu_device_memory(pmd_val(pmd))) BUG(); - return pfn_to_page((pmd_val(pmd) & SRMMU_PTD_PMASK) >> (PAGE_SHIFT-4)); + return pfn_to_page(pmd_pfn(pmd)); } -static inline unsigned long pgd_page_vaddr(pgd_t pgd) +static inline unsigned long __pmd_page(pmd_t pmd) { - if (srmmu_device_memory(pgd_val(pgd))) { - return ~0; + unsigned long v; + + if (srmmu_device_memory(pmd_val(pmd))) + BUG(); + + v = pmd_val(pmd) & SRMMU_PTD_PMASK; + return (unsigned long)__nocache_va(v << 4); +} + +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + unsigned long v = pmd_val(pmd) & SRMMU_PTD_PMASK; + return (unsigned long)__nocache_va(v << 4); +} + +static inline pmd_t *pud_pgtable(pud_t pud) +{ + if (srmmu_device_memory(pud_val(pud))) { + return (pmd_t *)~0; } else { - unsigned long v = pgd_val(pgd) & SRMMU_PTD_PMASK; - return (unsigned long)__nocache_va(v << 4); + unsigned long v = pud_val(pud) & SRMMU_PTD_PMASK; + return (pmd_t *)__nocache_va(v << 4); } } @@ -176,29 +182,27 @@ static inline int pmd_none(pmd_t pmd) static inline void pmd_clear(pmd_t *pmdp) { - int i; - for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) - set_pte((pte_t *)&pmdp->pmdv[i], __pte(0)); + set_pte((pte_t *)&pmd_val(*pmdp), __pte(0)); } -static inline int pgd_none(pgd_t pgd) +static inline int pud_none(pud_t pud) { - return !(pgd_val(pgd) & 0xFFFFFFF); + return !(pud_val(pud) & 0xFFFFFFF); } -static inline int pgd_bad(pgd_t pgd) +static inline int pud_bad(pud_t pud) { - return (pgd_val(pgd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; + return (pud_val(pud) & SRMMU_ET_MASK) != SRMMU_ET_PTD; } -static inline int pgd_present(pgd_t pgd) +static inline int pud_present(pud_t pud) { - return ((pgd_val(pgd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); + return ((pud_val(pud) & SRMMU_ET_MASK) == SRMMU_ET_PTD); } -static inline void pgd_clear(pgd_t *pgdp) +static inline void pud_clear(pud_t *pudp) { - set_pte((pte_t *)pgdp, __pte(0)); + set_pte((pte_t *)pudp, __pte(0)); } /* @@ -220,19 +224,6 @@ static inline int pte_young(pte_t pte) return pte_val(pte) & SRMMU_REF; } -/* - * The following only work if pte_present() is not true. - */ -static inline int pte_file(pte_t pte) -{ - return pte_val(pte) & SRMMU_FILE; -} - -static inline int pte_special(pte_t pte) -{ - return 0; -} - static inline pte_t pte_wrprotect(pte_t pte) { return __pte(pte_val(pte) & ~SRMMU_WRITE); @@ -248,7 +239,7 @@ static inline pte_t pte_mkold(pte_t pte) return __pte(pte_val(pte) & ~SRMMU_REF); } -static inline pte_t pte_mkwrite(pte_t pte) +static inline pte_t pte_mkwrite_novma(pte_t pte) { return __pte(pte_val(pte) | SRMMU_WRITE); } @@ -263,9 +254,12 @@ static inline pte_t pte_mkyoung(pte_t pte) return __pte(pte_val(pte) | SRMMU_REF); } -#define pte_mkspecial(pte) (pte) +#define PFN_PTE_SHIFT (PAGE_SHIFT - 4) -#define pfn_pte(pfn, prot) mk_pte(pfn_to_page(pfn), prot) +static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot) +{ + return __pte((pfn << PFN_PTE_SHIFT) | pgprot_val(pgprot)); +} static inline unsigned long pte_pfn(pte_t pte) { @@ -277,20 +271,11 @@ static inline unsigned long pte_pfn(pte_t pte) */ return ~0UL; } - return (pte_val(pte) & SRMMU_PTE_PMASK) >> (PAGE_SHIFT-4); + return (pte_val(pte) & SRMMU_PTE_PMASK) >> PFN_PTE_SHIFT; } #define pte_page(pte) pfn_to_page(pte_pfn(pte)) -/* - * Conversion functions: convert a page and protection to a page entry, - * and a page entry and page directory to the page they refer to. - */ -static inline pte_t mk_pte(struct page *page, pgprot_t pgprot) -{ - return __pte((page_to_pfn(page) << (PAGE_SHIFT-4)) | pgprot_val(pgprot)); -} - static inline pte_t mk_pte_phys(unsigned long page, pgprot_t pgprot) { return __pte(((page) >> 4) | pgprot_val(pgprot)); @@ -304,7 +289,7 @@ static inline pte_t mk_pte_io(unsigned long page, pgprot_t pgprot, int space) #define pgprot_noncached pgprot_noncached static inline pgprot_t pgprot_noncached(pgprot_t prot) { - prot &= ~__pgprot(SRMMU_CACHE); + pgprot_val(prot) &= ~pgprot_val(__pgprot(SRMMU_CACHE)); return prot; } @@ -315,29 +300,8 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) pgprot_val(newprot)); } -#define pgd_index(address) ((address) >> PGDIR_SHIFT) - -/* to find an entry in a page-table-directory */ -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) - -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - -/* Find an entry in the second-level page table.. */ -static inline pmd_t *pmd_offset(pgd_t * dir, unsigned long address) -{ - return (pmd_t *) pgd_page_vaddr(*dir) + - ((address >> PMD_SHIFT) & (PTRS_PER_PMD - 1)); -} - -/* Find an entry in the third-level page table.. */ -pte_t *pte_offset_kernel(pmd_t * dir, unsigned long address); - -/* - * This shortcut works on sun4m (and sun4d) because the nocache area is static. - */ -#define pte_offset_map(d, a) pte_offset_kernel(d,a) -#define pte_unmap(pte) do{}while(0) +/* only used by the huge vmap code, should never be called */ +#define pud_page(pud) NULL struct seq_file; void mmu_info(struct seq_file *m); @@ -348,12 +312,22 @@ void mmu_info(struct seq_file *m); #define FAULT_CODE_USER 0x4 #define update_mmu_cache(vma, address, ptep) do { } while (0) +#define update_mmu_cache_range(vmf, vma, address, ptep, nr) do { } while (0) void srmmu_mapiorange(unsigned int bus, unsigned long xpa, unsigned long xva, unsigned int len); void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len); -/* Encode and de-code a swap entry */ +/* + * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that + * are !pte_none() && !pte_present(). + * + * Format of swap PTEs: + * + * 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 + * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 + * <-------------- offset ---------------> < type -> E 0 0 0 0 0 0 + */ static inline unsigned long __swp_type(swp_entry_t entry) { return (entry.val >> SRMMU_SWP_TYPE_SHIFT) & SRMMU_SWP_TYPE_MASK; @@ -374,21 +348,20 @@ static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset) #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) -/* file-offset-in-pte helpers */ -static inline unsigned long pte_to_pgoff(pte_t pte) +static inline bool pte_swp_exclusive(pte_t pte) { - return pte_val(pte) >> SRMMU_PTE_FILE_SHIFT; + return pte_val(pte) & SRMMU_SWP_EXCLUSIVE; } -static inline pte_t pgoff_to_pte(unsigned long pgoff) +static inline pte_t pte_swp_mkexclusive(pte_t pte) { - return __pte((pgoff << SRMMU_PTE_FILE_SHIFT) | SRMMU_FILE); + return __pte(pte_val(pte) | SRMMU_SWP_EXCLUSIVE); } -/* - * This is made a constant because mm/fremap.c required a constant. - */ -#define PTE_FILE_MAX_BITS 24 +static inline pte_t pte_swp_clear_exclusive(pte_t pte) +{ + return __pte(pte_val(pte) & ~SRMMU_SWP_EXCLUSIVE); +} static inline unsigned long __get_phys (unsigned long addr) @@ -414,12 +387,6 @@ __get_iospace (unsigned long addr) } } -extern unsigned long *sparc_valid_addr_bitmap; - -/* Needs to be defined here and not in linux/mm.h, as it is arch dependent */ -#define kern_addr_valid(addr) \ - (test_bit(__pa((unsigned long)(addr))>>20, sparc_valid_addr_bitmap)) - /* * For sparc32&64, the pfn in io_remap_pfn_range() carries <iospace> in * its high 4 bits. These macros/functions put it there or get it from there. @@ -428,12 +395,8 @@ extern unsigned long *sparc_valid_addr_bitmap; #define GET_IOSPACE(pfn) (pfn >> (BITS_PER_LONG - 4)) #define GET_PFN(pfn) (pfn & 0x0fffffffUL) -extern int remap_pfn_range(struct vm_area_struct *, unsigned long, unsigned long, - unsigned long, pgprot_t); - -static inline int io_remap_pfn_range(struct vm_area_struct *vma, - unsigned long from, unsigned long pfn, - unsigned long size, pgprot_t prot) +static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, + unsigned long size) { unsigned long long offset, space, phys_base; @@ -441,34 +404,31 @@ static inline int io_remap_pfn_range(struct vm_area_struct *vma, space = GET_IOSPACE(pfn); phys_base = offset | (space << 32ULL); - return remap_pfn_range(vma, from, phys_base >> PAGE_SHIFT, size, prot); + return phys_base >> PAGE_SHIFT; } -#define io_remap_pfn_range io_remap_pfn_range +#define io_remap_pfn_range_pfn io_remap_pfn_range_pfn #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS #define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \ ({ \ int __changed = !pte_same(*(__ptep), __entry); \ if (__changed) { \ - set_pte_at((__vma)->vm_mm, (__address), __ptep, __entry); \ + set_pte(__ptep, __entry); \ flush_tlb_page(__vma, __address); \ } \ __changed; \ }) -#include <asm-generic/pgtable.h> - -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #define VMALLOC_START _AC(0xfe600000,UL) #define VMALLOC_END _AC(0xffc00000,UL) +#define MODULES_VADDR VMALLOC_START +#define MODULES_END VMALLOC_END /* We provide our own get_unmapped_area to cope with VA holes for userland */ #define HAVE_ARCH_UNMAPPED_AREA -/* - * No page table caches to initialise - */ -#define pgtable_cache_init() do { } while (0) +#define pmd_pgtable(pmd) ((pgtable_t)__pmd_page(pmd)) #endif /* !(_SPARC_PGTABLE_H) */ diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 36760317814f..615f460c50af 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * pgtable.h: SpitFire page table operations. * @@ -12,19 +13,20 @@ * the SpitFire page tables. */ +#include <asm-generic/pgtable-nop4d.h> #include <linux/compiler.h> #include <linux/const.h> #include <asm/types.h> #include <asm/spitfire.h> #include <asm/asi.h> +#include <asm/adi.h> #include <asm/page.h> #include <asm/processor.h> -#include <asm-generic/pgtable-nopud.h> - /* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB). * The page copy blockops can use 0x6000000 to 0x8000000. - * The TSB is mapped in the 0x8000000 to 0xa000000 range. + * The 8K TSB is mapped in the 0x8000000 to 0x8400000 range. + * The 4M TSB is mapped in the 0x8400000 to 0x8800000 range. * The PROM resides in an area spanning 0xf0000000 to 0x100000000. * The vmalloc area spans 0x100000000 to 0x200000000. * Since modules need to be in the lowest 32-bits of the address space, @@ -33,33 +35,43 @@ * 0x400000000. */ #define TLBTEMP_BASE _AC(0x0000000006000000,UL) -#define TSBMAP_BASE _AC(0x0000000008000000,UL) +#define TSBMAP_8K_BASE _AC(0x0000000008000000,UL) +#define TSBMAP_4M_BASE _AC(0x0000000008400000,UL) #define MODULES_VADDR _AC(0x0000000010000000,UL) #define MODULES_LEN _AC(0x00000000e0000000,UL) #define MODULES_END _AC(0x00000000f0000000,UL) #define LOW_OBP_ADDRESS _AC(0x00000000f0000000,UL) #define HI_OBP_ADDRESS _AC(0x0000000100000000,UL) #define VMALLOC_START _AC(0x0000000100000000,UL) -#define VMALLOC_END _AC(0x0000010000000000,UL) -#define VMEMMAP_BASE _AC(0x0000010000000000,UL) - -#define vmemmap ((struct page *)VMEMMAP_BASE) +#define VMEMMAP_BASE VMALLOC_END /* PMD_SHIFT determines the size of the area a second-level page * table can map */ -#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-4)) +#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3)) #define PMD_SIZE (_AC(1,UL) << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE-1)) -#define PMD_BITS (PAGE_SHIFT - 2) +#define PMD_BITS (PAGE_SHIFT - 3) + +/* PUD_SHIFT determines the size of the area a third-level page + * table can map + */ +#define PUD_SHIFT (PMD_SHIFT + PMD_BITS) +#define PUD_SIZE (_AC(1,UL) << PUD_SHIFT) +#define PUD_MASK (~(PUD_SIZE-1)) +#define PUD_BITS (PAGE_SHIFT - 3) -/* PGDIR_SHIFT determines what a third-level page table entry can map */ -#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-4) + PMD_BITS) +/* PGDIR_SHIFT determines what a fourth-level page table entry can map */ +#define PGDIR_SHIFT (PUD_SHIFT + PUD_BITS) #define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) -#define PGDIR_BITS (PAGE_SHIFT - 2) +#define PGDIR_BITS (PAGE_SHIFT - 3) + +#if (MAX_PHYS_ADDRESS_BITS > PGDIR_SHIFT + PGDIR_BITS) +#error MAX_PHYS_ADDRESS_BITS exceeds what kernel page tables can support +#endif -#if (PGDIR_SHIFT + PGDIR_BITS) != 44 +#if (PGDIR_SHIFT + PGDIR_BITS) != 53 #error Page table parameters do not cover virtual address space properly. #endif @@ -67,54 +79,41 @@ #error PMD_SHIFT must equal HPAGE_SHIFT for transparent huge pages. #endif -/* PMDs point to PTE tables which are 4K aligned. */ -#define PMD_PADDR _AC(0xfffffffe,UL) -#define PMD_PADDR_SHIFT _AC(11,UL) - -#define PMD_ISHUGE _AC(0x00000001,UL) - -/* This is the PMD layout when PMD_ISHUGE is set. With 4MB huge - * pages, this frees up a bunch of bits in the layout that we can - * use for the protection settings and software metadata. - */ -#define PMD_HUGE_PADDR _AC(0xfffff800,UL) -#define PMD_HUGE_PROTBITS _AC(0x000007ff,UL) -#define PMD_HUGE_PRESENT _AC(0x00000400,UL) -#define PMD_HUGE_WRITE _AC(0x00000200,UL) -#define PMD_HUGE_DIRTY _AC(0x00000100,UL) -#define PMD_HUGE_ACCESSED _AC(0x00000080,UL) -#define PMD_HUGE_EXEC _AC(0x00000040,UL) -#define PMD_HUGE_SPLITTING _AC(0x00000020,UL) +#ifndef __ASSEMBLER__ -/* PGDs point to PMD tables which are 8K aligned. */ -#define PGD_PADDR _AC(0xfffffffc,UL) -#define PGD_PADDR_SHIFT _AC(11,UL) +extern unsigned long VMALLOC_END; -#ifndef __ASSEMBLY__ +#define vmemmap ((struct page *)VMEMMAP_BASE) #include <linux/sched.h> +#include <asm/tlbflush.h> + +bool kern_addr_valid(unsigned long addr); /* Entries per page directory level. */ -#define PTRS_PER_PTE (1UL << (PAGE_SHIFT-4)) +#define PTRS_PER_PTE (1UL << (PAGE_SHIFT-3)) #define PTRS_PER_PMD (1UL << PMD_BITS) +#define PTRS_PER_PUD (1UL << PUD_BITS) #define PTRS_PER_PGD (1UL << PGDIR_BITS) -/* Kernel has a separate 44bit address space. */ -#define FIRST_USER_ADDRESS 0 - -#define pte_ERROR(e) __builtin_trap() -#define pmd_ERROR(e) __builtin_trap() -#define pgd_ERROR(e) __builtin_trap() +#define pmd_ERROR(e) \ + pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n", \ + __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0)) +#define pud_ERROR(e) \ + pr_err("%s:%d: bad pud %p(%016lx) seen at (%pS)\n", \ + __FILE__, __LINE__, &(e), pud_val(e), __builtin_return_address(0)) +#define pgd_ERROR(e) \ + pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n", \ + __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0)) -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ /* PTE bits which are the same in SUN4U and SUN4V format. */ #define _PAGE_VALID _AC(0x8000000000000000,UL) /* Valid TTE */ #define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/ #define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */ - -/* Advertise support for _PAGE_SPECIAL */ -#define __HAVE_ARCH_PTE_SPECIAL +#define _PAGE_PMD_HUGE _AC(0x0100000000000000,UL) /* Huge page */ +#define _PAGE_PUD_HUGE _PAGE_PMD_HUGE /* SUN4U pte bits... */ #define _PAGE_SZ4MB_4U _AC(0x6000000000000000,UL) /* 4MB Page */ @@ -125,6 +124,7 @@ #define _PAGE_IE_4U _AC(0x0800000000000000,UL) /* Invert Endianness */ #define _PAGE_SOFT2_4U _AC(0x07FC000000000000,UL) /* Software bits, set 2 */ #define _PAGE_SPECIAL_4U _AC(0x0200000000000000,UL) /* Special page */ +#define _PAGE_PMD_HUGE_4U _AC(0x0100000000000000,UL) /* Huge page */ #define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */ #define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */ #define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */ @@ -135,7 +135,6 @@ #define _PAGE_SOFT_4U _AC(0x0000000000001F80,UL) /* Software bits: */ #define _PAGE_EXEC_4U _AC(0x0000000000001000,UL) /* Executable SW bit */ #define _PAGE_MODIFIED_4U _AC(0x0000000000000800,UL) /* Modified (dirty) */ -#define _PAGE_FILE_4U _AC(0x0000000000000800,UL) /* Pagecache page */ #define _PAGE_ACCESSED_4U _AC(0x0000000000000400,UL) /* Accessed (ref'd) */ #define _PAGE_READ_4U _AC(0x0000000000000200,UL) /* Readable SW Bit */ #define _PAGE_WRITE_4U _AC(0x0000000000000100,UL) /* Writable SW Bit */ @@ -155,16 +154,18 @@ #define _PAGE_READ_4V _AC(0x0800000000000000,UL) /* Readable SW Bit */ #define _PAGE_WRITE_4V _AC(0x0400000000000000,UL) /* Writable SW Bit */ #define _PAGE_SPECIAL_4V _AC(0x0200000000000000,UL) /* Special page */ +#define _PAGE_PMD_HUGE_4V _AC(0x0100000000000000,UL) /* Huge page */ #define _PAGE_PADDR_4V _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13] */ #define _PAGE_IE_4V _AC(0x0000000000001000,UL) /* Invert Endianness */ #define _PAGE_E_4V _AC(0x0000000000000800,UL) /* side-Effect */ #define _PAGE_CP_4V _AC(0x0000000000000400,UL) /* Cacheable in P-Cache */ #define _PAGE_CV_4V _AC(0x0000000000000200,UL) /* Cacheable in V-Cache */ +/* Bit 9 is used to enable MCD corruption detection instead on M7 */ +#define _PAGE_MCD_4V _AC(0x0000000000000200,UL) /* Memory Corruption */ #define _PAGE_P_4V _AC(0x0000000000000100,UL) /* Privileged Page */ #define _PAGE_EXEC_4V _AC(0x0000000000000080,UL) /* Executable Page */ #define _PAGE_W_4V _AC(0x0000000000000040,UL) /* Writable */ #define _PAGE_SOFT_4V _AC(0x0000000000000030,UL) /* Software bits */ -#define _PAGE_FILE_4V _AC(0x0000000000000020,UL) /* Pagecache page */ #define _PAGE_PRESENT_4V _AC(0x0000000000000010,UL) /* Present */ #define _PAGE_RESV_4V _AC(0x0000000000000008,UL) /* Reserved */ #define _PAGE_SZ16GB_4V _AC(0x0000000000000007,UL) /* 16GB Page */ @@ -180,40 +181,28 @@ #define _PAGE_SZBITS_4U _PAGE_SZ8K_4U #define _PAGE_SZBITS_4V _PAGE_SZ8K_4V +#if REAL_HPAGE_SHIFT != 22 +#error REAL_HPAGE_SHIFT and _PAGE_SZHUGE_foo must match up +#endif + #define _PAGE_SZHUGE_4U _PAGE_SZ4MB_4U #define _PAGE_SZHUGE_4V _PAGE_SZ4MB_4V -/* These are actually filled in at boot time by sun4{u,v}_pgprot_init() */ -#define __P000 __pgprot(0) -#define __P001 __pgprot(0) -#define __P010 __pgprot(0) -#define __P011 __pgprot(0) -#define __P100 __pgprot(0) -#define __P101 __pgprot(0) -#define __P110 __pgprot(0) -#define __P111 __pgprot(0) +/* We borrow bit 20 to store the exclusive marker in swap PTEs. */ +#define _PAGE_SWP_EXCLUSIVE _AC(0x0000000000100000, UL) -#define __S000 __pgprot(0) -#define __S001 __pgprot(0) -#define __S010 __pgprot(0) -#define __S011 __pgprot(0) -#define __S100 __pgprot(0) -#define __S101 __pgprot(0) -#define __S110 __pgprot(0) -#define __S111 __pgprot(0) +#ifndef __ASSEMBLER__ -#ifndef __ASSEMBLY__ +pte_t mk_pte_io(unsigned long, pgprot_t, int, unsigned long); -extern pte_t mk_pte_io(unsigned long, pgprot_t, int, unsigned long); - -extern unsigned long pte_sz_bits(unsigned long size); +unsigned long pte_sz_bits(unsigned long size); extern pgprot_t PAGE_KERNEL; extern pgprot_t PAGE_KERNEL_LOCKED; extern pgprot_t PAGE_COPY; extern pgprot_t PAGE_SHARED; -/* XXX This uglyness is for the atyfb driver's sparc mmap() support. XXX */ +/* XXX This ugliness is for the atyfb driver's sparc mmap() support. XXX */ extern unsigned long _PAGE_IE; extern unsigned long _PAGE_E; extern unsigned long _PAGE_CACHE; @@ -236,18 +225,13 @@ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) BUILD_BUG_ON(_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL); return __pte(paddr | pgprot_val(prot)); } -#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) #ifdef CONFIG_TRANSPARENT_HUGEPAGE -extern pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot); -#define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot)) - -extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot); - -static inline pmd_t pmd_mkhuge(pmd_t pmd) +static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) { - /* Do nothing, mk_pmd() does this part. */ - return pmd; + pte_t pte = pfn_pte(page_nr, pgprot); + + return __pmd(pte_val(pte)); } #endif @@ -277,8 +261,8 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t prot) { unsigned long mask, tmp; - /* SUN4U: 0x600307ffffffecb8 (negated == 0x9ffcf80000001347) - * SUN4V: 0x30ffffffffffee17 (negated == 0xcf000000000011e8) + /* SUN4U: 0x630107ffffffec38 (negated == 0x9cfef800000013c7) + * SUN4V: 0x33ffffffffffee07 (negated == 0xcc000000000011f8) * * Even if we use negation tricks the result is still a 6 * instruction sequence, so don't try to play fancy and just @@ -306,32 +290,41 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t prot) " sllx %1, 32, %1\n" " or %0, %1, %0\n" " .previous\n" + " .section .sun_m7_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%4), %1\n" + " sethi %%hi(%4), %0\n" + " .word 662b\n" + " or %1, %%ulo(%4), %1\n" + " or %0, %%lo(%4), %0\n" + " .word 663b\n" + " sllx %1, 32, %1\n" + " or %0, %1, %0\n" + " .previous\n" : "=r" (mask), "=r" (tmp) : "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U | - _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | _PAGE_PRESENT_4U | - _PAGE_SPECIAL), + _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | + _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4U), + "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V | + _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | + _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V), "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V | - _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | _PAGE_PRESENT_4V | - _PAGE_SPECIAL)); + _PAGE_CP_4V | _PAGE_E_4V | + _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V)); return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask)); } -static inline pte_t pgoff_to_pte(unsigned long off) +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { - off <<= PAGE_SHIFT; + pte_t pte = __pte(pmd_val(pmd)); - __asm__ __volatile__( - "\n661: or %0, %2, %0\n" - " .section .sun4v_1insn_patch, \"ax\"\n" - " .word 661b\n" - " or %0, %3, %0\n" - " .previous\n" - : "=r" (off) - : "0" (off), "i" (_PAGE_FILE_4U), "i" (_PAGE_FILE_4V)); + pte = pte_modify(pte, newprot); - return __pte(off); + return __pmd(pte_val(pte)); } +#endif static inline pgprot_t pgprot_noncached(pgprot_t prot) { @@ -345,9 +338,15 @@ static inline pgprot_t pgprot_noncached(pgprot_t prot) " andn %0, %4, %0\n" " or %0, %5, %0\n" " .previous\n" + " .section .sun_m7_2insn_patch, \"ax\"\n" + " .word 661b\n" + " andn %0, %6, %0\n" + " or %0, %5, %0\n" + " .previous\n" : "=r" (val) : "0" (val), "i" (_PAGE_CP_4U | _PAGE_CV_4U), "i" (_PAGE_E_4U), - "i" (_PAGE_CP_4V | _PAGE_CV_4V), "i" (_PAGE_E_4V)); + "i" (_PAGE_CP_4V | _PAGE_CV_4V), "i" (_PAGE_E_4V), + "i" (_PAGE_CP_4V)); return __pgprot(val); } @@ -357,8 +356,46 @@ static inline pgprot_t pgprot_noncached(pgprot_t prot) */ #define pgprot_noncached pgprot_noncached -#ifdef CONFIG_HUGETLB_PAGE -static inline pte_t pte_mkhuge(pte_t pte) +static inline unsigned long pte_dirty(pte_t pte) +{ + unsigned long mask; + + __asm__ __volatile__( + "\n661: mov %1, %0\n" + " nop\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%2), %0\n" + " sllx %0, 32, %0\n" + " .previous\n" + : "=r" (mask) + : "i" (_PAGE_MODIFIED_4U), "i" (_PAGE_MODIFIED_4V)); + + return (pte_val(pte) & mask); +} + +static inline unsigned long pte_write(pte_t pte) +{ + unsigned long mask; + + __asm__ __volatile__( + "\n661: mov %1, %0\n" + " nop\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%2), %0\n" + " sllx %0, 32, %0\n" + " .previous\n" + : "=r" (mask) + : "i" (_PAGE_WRITE_4U), "i" (_PAGE_WRITE_4V)); + + return (pte_val(pte) & mask); +} + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags); +#define arch_make_huge_pte arch_make_huge_pte +static inline unsigned long __pte_default_huge_mask(void) { unsigned long mask; @@ -373,32 +410,86 @@ static inline pte_t pte_mkhuge(pte_t pte) : "=r" (mask) : "i" (_PAGE_SZHUGE_4U), "i" (_PAGE_SZHUGE_4V)); - return __pte(pte_val(pte) | mask); + return mask; +} + +static inline pte_t pte_mkhuge(pte_t pte) +{ + return __pte(pte_val(pte) | __pte_default_huge_mask()); +} + +static inline bool is_default_hugetlb_pte(pte_t pte) +{ + unsigned long mask = __pte_default_huge_mask(); + + return (pte_val(pte) & mask) == mask; +} + +static inline bool is_hugetlb_pmd(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_PMD_HUGE); +} + +static inline bool is_hugetlb_pud(pud_t pud) +{ + return !!(pud_val(pud) & _PAGE_PUD_HUGE); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline pmd_t pmd_mkhuge(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkhuge(pte); + pte_val(pte) |= _PAGE_PMD_HUGE; + + return __pmd(pte_val(pte)); +} +#endif +#else +static inline bool is_hugetlb_pte(pte_t pte) +{ + return false; } #endif +static inline pte_t __pte_mkhwwrite(pte_t pte) +{ + unsigned long val = pte_val(pte); + + /* + * Note: we only want to set the HW writable bit if the SW writable bit + * and the SW dirty bit are set. + */ + __asm__ __volatile__( + "\n661: or %0, %2, %0\n" + " .section .sun4v_1insn_patch, \"ax\"\n" + " .word 661b\n" + " or %0, %3, %0\n" + " .previous\n" + : "=r" (val) + : "0" (val), "i" (_PAGE_W_4U), "i" (_PAGE_W_4V)); + + return __pte(val); +} + static inline pte_t pte_mkdirty(pte_t pte) { - unsigned long val = pte_val(pte), tmp; + unsigned long val = pte_val(pte), mask; __asm__ __volatile__( - "\n661: or %0, %3, %0\n" - " nop\n" - "\n662: nop\n" + "\n661: mov %1, %0\n" " nop\n" " .section .sun4v_2insn_patch, \"ax\"\n" " .word 661b\n" - " sethi %%uhi(%4), %1\n" - " sllx %1, 32, %1\n" - " .word 662b\n" - " or %1, %%lo(%4), %1\n" - " or %0, %1, %0\n" + " sethi %%uhi(%2), %0\n" + " sllx %0, 32, %0\n" " .previous\n" - : "=r" (val), "=r" (tmp) - : "0" (val), "i" (_PAGE_MODIFIED_4U | _PAGE_W_4U), - "i" (_PAGE_MODIFIED_4V | _PAGE_W_4V)); + : "=r" (mask) + : "i" (_PAGE_MODIFIED_4U), "i" (_PAGE_MODIFIED_4V)); - return __pte(val); + pte = __pte(val | mask); + return pte_write(pte) ? __pte_mkhwwrite(pte) : pte; } static inline pte_t pte_mkclean(pte_t pte) @@ -425,7 +516,7 @@ static inline pte_t pte_mkclean(pte_t pte) return __pte(val); } -static inline pte_t pte_mkwrite(pte_t pte) +static inline pte_t pte_mkwrite_novma(pte_t pte) { unsigned long val = pte_val(pte), mask; @@ -440,7 +531,8 @@ static inline pte_t pte_mkwrite(pte_t pte) : "=r" (mask) : "i" (_PAGE_WRITE_4U), "i" (_PAGE_WRITE_4V)); - return __pte(val | mask); + pte = __pte(val | mask); + return pte_dirty(pte) ? __pte_mkhwwrite(pte) : pte; } static inline pte_t pte_wrprotect(pte_t pte) @@ -513,43 +605,19 @@ static inline pte_t pte_mkspecial(pte_t pte) return pte; } -static inline unsigned long pte_young(pte_t pte) +static inline pte_t pte_mkmcd(pte_t pte) { - unsigned long mask; - - __asm__ __volatile__( - "\n661: mov %1, %0\n" - " nop\n" - " .section .sun4v_2insn_patch, \"ax\"\n" - " .word 661b\n" - " sethi %%uhi(%2), %0\n" - " sllx %0, 32, %0\n" - " .previous\n" - : "=r" (mask) - : "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V)); - - return (pte_val(pte) & mask); + pte_val(pte) |= _PAGE_MCD_4V; + return pte; } -static inline unsigned long pte_dirty(pte_t pte) +static inline pte_t pte_mknotmcd(pte_t pte) { - unsigned long mask; - - __asm__ __volatile__( - "\n661: mov %1, %0\n" - " nop\n" - " .section .sun4v_2insn_patch, \"ax\"\n" - " .word 661b\n" - " sethi %%uhi(%2), %0\n" - " sllx %0, 32, %0\n" - " .previous\n" - : "=r" (mask) - : "i" (_PAGE_MODIFIED_4U), "i" (_PAGE_MODIFIED_4V)); - - return (pte_val(pte) & mask); + pte_val(pte) &= ~_PAGE_MCD_4V; + return pte; } -static inline unsigned long pte_write(pte_t pte) +static inline unsigned long pte_young(pte_t pte) { unsigned long mask; @@ -562,7 +630,7 @@ static inline unsigned long pte_write(pte_t pte) " sllx %0, 32, %0\n" " .previous\n" : "=r" (mask) - : "i" (_PAGE_WRITE_4U), "i" (_PAGE_WRITE_4V)); + : "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V)); return (pte_val(pte) & mask); } @@ -583,22 +651,6 @@ static inline unsigned long pte_exec(pte_t pte) return (pte_val(pte) & mask); } -static inline unsigned long pte_file(pte_t pte) -{ - unsigned long val = pte_val(pte); - - __asm__ __volatile__( - "\n661: and %0, %2, %0\n" - " .section .sun4v_1insn_patch, \"ax\"\n" - " .word 661b\n" - " and %0, %3, %0\n" - " .previous\n" - : "=r" (val) - : "0" (val), "i" (_PAGE_FILE_4U), "i" (_PAGE_FILE_4V)); - - return val; -} - static inline unsigned long pte_present(pte_t pte) { unsigned long val = pte_val(pte); @@ -616,7 +668,7 @@ static inline unsigned long pte_present(pte_t pte) } #define pte_accessible pte_accessible -static inline unsigned long pte_accessible(pte_t a) +static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a) { return pte_val(a) & _PAGE_VALID; } @@ -626,98 +678,144 @@ static inline unsigned long pte_special(pte_t pte) return pte_val(pte) & _PAGE_SPECIAL; } -static inline int pmd_large(pmd_t pmd) +#define pmd_leaf pmd_leaf +static inline bool pmd_leaf(pmd_t pmd) { - return (pmd_val(pmd) & (PMD_ISHUGE | PMD_HUGE_PRESENT)) == - (PMD_ISHUGE | PMD_HUGE_PRESENT); + pte_t pte = __pte(pmd_val(pmd)); + + return pte_val(pte) & _PAGE_PMD_HUGE; } -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -static inline int pmd_young(pmd_t pmd) +static inline unsigned long pmd_pfn(pmd_t pmd) { - return pmd_val(pmd) & PMD_HUGE_ACCESSED; + pte_t pte = __pte(pmd_val(pmd)); + + return pte_pfn(pte); } -static inline int pmd_write(pmd_t pmd) +#define pmd_write pmd_write +static inline unsigned long pmd_write(pmd_t pmd) { - return pmd_val(pmd) & PMD_HUGE_WRITE; + pte_t pte = __pte(pmd_val(pmd)); + + return pte_write(pte); } -static inline unsigned long pmd_pfn(pmd_t pmd) +#define pud_write(pud) pte_write(__pte(pud_val(pud))) + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define pmd_dirty pmd_dirty +static inline unsigned long pmd_dirty(pmd_t pmd) { - unsigned long val = pmd_val(pmd) & PMD_HUGE_PADDR; + pte_t pte = __pte(pmd_val(pmd)); - return val >> (PAGE_SHIFT - PMD_PADDR_SHIFT); + return pte_dirty(pte); } -static inline int pmd_trans_splitting(pmd_t pmd) +#define pmd_young pmd_young +static inline unsigned long pmd_young(pmd_t pmd) { - return (pmd_val(pmd) & (PMD_ISHUGE|PMD_HUGE_SPLITTING)) == - (PMD_ISHUGE|PMD_HUGE_SPLITTING); + pte_t pte = __pte(pmd_val(pmd)); + + return pte_young(pte); } -static inline int pmd_trans_huge(pmd_t pmd) +static inline unsigned long pmd_trans_huge(pmd_t pmd) { - return pmd_val(pmd) & PMD_ISHUGE; -} + pte_t pte = __pte(pmd_val(pmd)); -#define has_transparent_hugepage() 1 + return pte_val(pte) & _PAGE_PMD_HUGE; +} static inline pmd_t pmd_mkold(pmd_t pmd) { - pmd_val(pmd) &= ~PMD_HUGE_ACCESSED; - return pmd; + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkold(pte); + + return __pmd(pte_val(pte)); } static inline pmd_t pmd_wrprotect(pmd_t pmd) { - pmd_val(pmd) &= ~PMD_HUGE_WRITE; - return pmd; + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_wrprotect(pte); + + return __pmd(pte_val(pte)); } static inline pmd_t pmd_mkdirty(pmd_t pmd) { - pmd_val(pmd) |= PMD_HUGE_DIRTY; - return pmd; + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkdirty(pte); + + return __pmd(pte_val(pte)); } -static inline pmd_t pmd_mkyoung(pmd_t pmd) +static inline pmd_t pmd_mkclean(pmd_t pmd) { - pmd_val(pmd) |= PMD_HUGE_ACCESSED; - return pmd; + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkclean(pte); + + return __pmd(pte_val(pte)); } -static inline pmd_t pmd_mkwrite(pmd_t pmd) +static inline pmd_t pmd_mkyoung(pmd_t pmd) { - pmd_val(pmd) |= PMD_HUGE_WRITE; - return pmd; + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkyoung(pte); + + return __pmd(pte_val(pte)); } -static inline pmd_t pmd_mknotpresent(pmd_t pmd) +static inline pmd_t pmd_mkwrite_novma(pmd_t pmd) { - pmd_val(pmd) &= ~PMD_HUGE_PRESENT; - return pmd; + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkwrite_novma(pte); + + return __pmd(pte_val(pte)); } -static inline pmd_t pmd_mksplitting(pmd_t pmd) +#define pmd_pgprot pmd_pgprot +static inline pgprot_t pmd_pgprot(pmd_t entry) { - pmd_val(pmd) |= PMD_HUGE_SPLITTING; - return pmd; -} + unsigned long val = pmd_val(entry); -extern pgprot_t pmd_pgprot(pmd_t entry); + return __pgprot(val); +} #endif static inline int pmd_present(pmd_t pmd) { - return pmd_val(pmd) != 0U; + return pmd_val(pmd) != 0UL; } #define pmd_none(pmd) (!pmd_val(pmd)) +/* pmd_bad() is only called on non-trans-huge PMDs. Our encoding is + * very simple, it's just the physical address. PTE tables are of + * size PAGE_SIZE so make sure the sub-PAGE_SIZE bits are clear and + * the top bits outside of the range of any physical address size we + * support are clear as well. We also validate the physical itself. + */ +#define pmd_bad(pmd) (pmd_val(pmd) & ~PAGE_MASK) + +#define pud_none(pud) (!pud_val(pud)) + +#define pud_bad(pud) (pud_val(pud) & ~PAGE_MASK) + +#define p4d_none(p4d) (!p4d_val(p4d)) + +#define p4d_bad(p4d) (p4d_val(p4d) & ~PAGE_MASK) + #ifdef CONFIG_TRANSPARENT_HUGEPAGE -extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, - pmd_t *pmdp, pmd_t pmd); +void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd); #else static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) @@ -728,68 +826,97 @@ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, static inline void pmd_set(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) { - unsigned long val = __pa((unsigned long) (ptep)) >> PMD_PADDR_SHIFT; + unsigned long val = __pa((unsigned long) (ptep)); pmd_val(*pmdp) = val; } #define pud_set(pudp, pmdp) \ - (pud_val(*(pudp)) = (__pa((unsigned long) (pmdp)) >> PGD_PADDR_SHIFT)) -static inline unsigned long __pmd_page(pmd_t pmd) + (pud_val(*(pudp)) = (__pa((unsigned long) (pmdp)))) +static inline unsigned long pmd_page_vaddr(pmd_t pmd) { - unsigned long paddr = (unsigned long) pmd_val(pmd); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (pmd_val(pmd) & PMD_ISHUGE) - paddr &= PMD_HUGE_PADDR; -#endif - paddr <<= PMD_PADDR_SHIFT; - return ((unsigned long) __va(paddr)); -} -#define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd)) -#define pud_page_vaddr(pud) \ - ((unsigned long) __va((((unsigned long)pud_val(pud))<<PGD_PADDR_SHIFT))) -#define pud_page(pud) virt_to_page((void *)pud_page_vaddr(pud)) -#define pmd_bad(pmd) (0) -#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0U) -#define pud_none(pud) (!pud_val(pud)) -#define pud_bad(pud) (0) + pte_t pte = __pte(pmd_val(pmd)); + unsigned long pfn; + + pfn = pte_pfn(pte); + + return ((unsigned long) __va(pfn << PAGE_SHIFT)); +} + +static inline pmd_t *pud_pgtable(pud_t pud) +{ + pte_t pte = __pte(pud_val(pud)); + unsigned long pfn; + + pfn = pte_pfn(pte); + + return ((pmd_t *) __va(pfn << PAGE_SHIFT)); +} + +#define pmd_page(pmd) virt_to_page((void *)pmd_page_vaddr(pmd)) +#define pud_page(pud) virt_to_page((void *)pud_pgtable(pud)) +#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL) #define pud_present(pud) (pud_val(pud) != 0U) -#define pud_clear(pudp) (pud_val(*(pudp)) = 0U) +#define pud_clear(pudp) (pud_val(*(pudp)) = 0UL) +#define p4d_pgtable(p4d) \ + ((pud_t *) __va(p4d_val(p4d))) +#define p4d_present(p4d) (p4d_val(p4d) != 0U) +#define p4d_clear(p4dp) (p4d_val(*(p4dp)) = 0UL) -/* Same in both SUN4V and SUN4U. */ -#define pte_none(pte) (!pte_val(pte)) +/* only used by the stubbed out hugetlb gup code, should never be called */ +#define p4d_page(p4d) NULL + +#define pud_leaf pud_leaf +static inline bool pud_leaf(pud_t pud) +{ + pte_t pte = __pte(pud_val(pud)); + + return pte_val(pte) & _PAGE_PMD_HUGE; +} -/* to find an entry in a page-table-directory. */ -#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) +#define pud_pfn pud_pfn +static inline unsigned long pud_pfn(pud_t pud) +{ + pte_t pte = __pte(pud_val(pud)); + + return pte_pfn(pte); +} -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) +/* Same in both SUN4V and SUN4U. */ +#define pte_none(pte) (!pte_val(pte)) -/* Find an entry in the second-level page table.. */ -#define pmd_offset(pudp, address) \ - ((pmd_t *) pud_page_vaddr(*(pudp)) + \ - (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))) +#define p4d_set(p4dp, pudp) \ + (p4d_val(*(p4dp)) = (__pa((unsigned long) (pudp)))) -/* Find an entry in the third-level page table.. */ -#define pte_index(dir, address) \ - ((pte_t *) __pmd_page(*(dir)) + \ - ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))) -#define pte_offset_kernel pte_index -#define pte_offset_map pte_index -#define pte_unmap(pte) do { } while (0) +/* We cannot include <linux/mm_types.h> at this point yet: */ +extern struct mm_struct init_mm; /* Actual page table PTE updates. */ -extern void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, - pte_t *ptep, pte_t orig, int fullmm); +void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, + pte_t *ptep, pte_t orig, int fullmm, + unsigned int hugepage_shift); -#define __HAVE_ARCH_PMDP_GET_AND_CLEAR -static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, - unsigned long addr, - pmd_t *pmdp) +static void maybe_tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, + pte_t *ptep, pte_t orig, int fullmm, + unsigned int hugepage_shift) +{ + /* It is more efficient to let flush_tlb_kernel_range() + * handle init_mm tlb flushes. + * + * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U + * and SUN4V pte layout, so this inline test is fine. + */ + if (likely(mm != &init_mm) && pte_accessible(mm, orig)) + tlb_batch_add(mm, vaddr, ptep, orig, fullmm, hugepage_shift); +} + +#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR +static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, + unsigned long addr, + pmd_t *pmdp) { pmd_t pmd = *pmdp; - set_pmd_at(mm, addr, pmdp, __pmd(0U)); + set_pmd_at(mm, addr, pmdp, __pmd(0UL)); return pmd; } @@ -799,19 +926,24 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t orig = *ptep; *ptep = pte; - - /* It is more efficient to let flush_tlb_kernel_range() - * handle init_mm tlb flushes. - * - * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U - * and SUN4V pte layout, so this inline test is fine. - */ - if (likely(mm != &init_mm) && pte_accessible(orig)) - tlb_batch_add(mm, addr, ptep, orig, fullmm); + maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm, PAGE_SHIFT); } -#define set_pte_at(mm,addr,ptep,pte) \ - __set_pte_at((mm), (addr), (ptep), (pte), 0) +#define PFN_PTE_SHIFT PAGE_SHIFT + +static inline void set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + for (;;) { + __set_pte_at(mm, addr, ptep, pte, 0); + if (--nr == 0) + break; + ptep++; + pte_val(pte) += PAGE_SIZE; + addr += PAGE_SIZE; + } +} +#define set_ptes set_ptes #define pte_clear(mm,addr,ptep) \ set_pte_at((mm), (addr), (ptep), __pte(0UL)) @@ -822,7 +954,7 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, #ifdef DCACHE_ALIASING_POSSIBLE #define __HAVE_ARCH_MOVE_PTE -#define move_pte(pte, prot, old_addr, new_addr) \ +#define move_pte(pte, old_addr, new_addr) \ ({ \ pte_t newpte = (pte); \ if (tlb_type != hypervisor && pte_present(pte)) { \ @@ -830,67 +962,83 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, \ if (pfn_valid(this_pfn) && \ (((old_addr) ^ (new_addr)) & (1 << 13))) \ - flush_dcache_page_all(current->mm, \ - pfn_to_page(this_pfn)); \ + flush_dcache_folio_all(current->mm, \ + page_folio(pfn_to_page(this_pfn))); \ } \ newpte; \ }) #endif -extern pgd_t swapper_pg_dir[2048]; -extern pmd_t swapper_low_pmd_dir[2048]; +extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; -extern void paging_init(void); -extern unsigned long find_ecache_flush_span(unsigned long size); +void paging_init(void); +unsigned long find_ecache_flush_span(unsigned long size); struct seq_file; -extern void mmu_info(struct seq_file *); +void mmu_info(struct seq_file *); struct vm_area_struct; -extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *); +void update_mmu_cache_range(struct vm_fault *, struct vm_area_struct *, + unsigned long addr, pte_t *ptep, unsigned int nr); +#define update_mmu_cache(vma, addr, ptep) \ + update_mmu_cache_range(NULL, vma, addr, ptep, 1) #ifdef CONFIG_TRANSPARENT_HUGEPAGE -extern void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, - pmd_t *pmd); +void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, + pmd_t *pmd); + +#define __HAVE_ARCH_PMDP_INVALIDATE +extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmdp); #define __HAVE_ARCH_PGTABLE_DEPOSIT -extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, - pgtable_t pgtable); +void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, + pgtable_t pgtable); #define __HAVE_ARCH_PGTABLE_WITHDRAW -extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); +pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); #endif -/* Encode and de-code a swap entry */ -#define __swp_type(entry) (((entry).val >> PAGE_SHIFT) & 0xffUL) +/* + * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that + * are !pte_none() && !pte_present(). + * + * Format of swap PTEs: + * + * 6 6 6 6 5 5 5 5 5 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3 3 + * 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 + * <--------------------------- offset --------------------------- + * + * 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 + * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 + * --------------------> E <-- type ---> <------- zeroes --------> + */ +#define __swp_type(entry) (((entry).val >> PAGE_SHIFT) & 0x7fUL) #define __swp_offset(entry) ((entry).val >> (PAGE_SHIFT + 8UL)) #define __swp_entry(type, offset) \ ( (swp_entry_t) \ { \ - (((long)(type) << PAGE_SHIFT) | \ + ((((long)(type) & 0x7fUL) << PAGE_SHIFT) | \ ((long)(offset) << (PAGE_SHIFT + 8UL))) \ } ) #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) -/* File offset in PTE support. */ -extern unsigned long pte_file(pte_t); -#define pte_to_pgoff(pte) (pte_val(pte) >> PAGE_SHIFT) -extern pte_t pgoff_to_pte(unsigned long); -#define PTE_FILE_MAX_BITS (64UL - PAGE_SHIFT - 1UL) - -extern unsigned long sparc64_valid_addr_bitmap[]; +static inline bool pte_swp_exclusive(pte_t pte) +{ + return pte_val(pte) & _PAGE_SWP_EXCLUSIVE; +} -/* Needs to be defined here and not in linux/mm.h, as it is arch dependent */ -static inline bool kern_addr_valid(unsigned long addr) +static inline pte_t pte_swp_mkexclusive(pte_t pte) { - unsigned long paddr = __pa(addr); + return __pte(pte_val(pte) | _PAGE_SWP_EXCLUSIVE); +} - if ((paddr >> 41UL) != 0UL) - return false; - return test_bit(paddr >> 22, sparc64_valid_addr_bitmap); +static inline pte_t pte_swp_clear_exclusive(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_SWP_EXCLUSIVE); } -extern int page_in_phys_avail(unsigned long paddr); +int page_in_phys_avail(unsigned long paddr); /* * For sparc32&64, the pfn in io_remap_pfn_range() carries <iospace> in @@ -900,12 +1048,41 @@ extern int page_in_phys_avail(unsigned long paddr); #define GET_IOSPACE(pfn) (pfn >> (BITS_PER_LONG - 4)) #define GET_PFN(pfn) (pfn & 0x0fffffffffffffffUL) -extern int remap_pfn_range(struct vm_area_struct *, unsigned long, unsigned long, - unsigned long, pgprot_t); +void adi_restore_tags(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, pte_t pte); + +int adi_save_tags(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, pte_t oldpte); + +#define __HAVE_ARCH_DO_SWAP_PAGE +static inline void arch_do_swap_page(struct mm_struct *mm, + struct vm_area_struct *vma, + unsigned long addr, + pte_t pte, pte_t oldpte) +{ + /* If this is a new page being mapped in, there can be no + * ADI tags stored away for this page. Skip looking for + * stored tags + */ + if (pte_none(oldpte)) + return; + + if (adi_state.enabled && (pte_val(pte) & _PAGE_MCD_4V)) + adi_restore_tags(mm, vma, addr, pte); +} + +#define __HAVE_ARCH_UNMAP_ONE +static inline int arch_unmap_one(struct mm_struct *mm, + struct vm_area_struct *vma, + unsigned long addr, pte_t oldpte) +{ + if (adi_state.enabled && (pte_val(oldpte) & _PAGE_MCD_4V)) + return adi_save_tags(mm, vma, addr, oldpte); + return 0; +} -static inline int io_remap_pfn_range(struct vm_area_struct *vma, - unsigned long from, unsigned long pfn, - unsigned long size, pgprot_t prot) +static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, + unsigned long size) { unsigned long offset = GET_PFN(pfn) << PAGE_SHIFT; int space = GET_IOSPACE(pfn); @@ -913,12 +1090,50 @@ static inline int io_remap_pfn_range(struct vm_area_struct *vma, phys_base = offset | (((unsigned long) space) << 32UL); - return remap_pfn_range(vma, from, phys_base >> PAGE_SHIFT, size, prot); + return phys_base >> PAGE_SHIFT; } -#define io_remap_pfn_range io_remap_pfn_range +#define io_remap_pfn_range_pfn io_remap_pfn_range_pfn -#include <asm/tlbflush.h> -#include <asm-generic/pgtable.h> +static inline unsigned long __untagged_addr(unsigned long start) +{ + if (adi_capable()) { + long addr = start; + + /* If userspace has passed a versioned address, kernel + * will not find it in the VMAs since it does not store + * the version tags in the list of VMAs. Storing version + * tags in list of VMAs is impractical since they can be + * changed any time from userspace without dropping into + * kernel. Any address search in VMAs will be done with + * non-versioned addresses. Ensure the ADI version bits + * are dropped here by sign extending the last bit before + * ADI bits. IOMMU does not implement version tags. + */ + return (addr << (long)adi_nbits()) >> (long)adi_nbits(); + } + + return start; +} +#define untagged_addr(addr) \ + ((__typeof__(addr))(__untagged_addr((unsigned long)(addr)))) + +static inline bool pte_access_permitted(pte_t pte, bool write) +{ + u64 prot; + + if (tlb_type == hypervisor) { + prot = _PAGE_PRESENT_4V | _PAGE_P_4V; + if (write) + prot |= _PAGE_WRITE_4V; + } else { + prot = _PAGE_PRESENT_4U | _PAGE_P_4U; + if (write) + prot |= _PAGE_WRITE_4U; + } + + return (pte_val(pte) & (prot | _PAGE_SPECIAL)) == prot; +} +#define pte_access_permitted pte_access_permitted /* We provide our own get_unmapped_area to cope with VA holes and * SHM area cache aliasing for userland. @@ -929,21 +1144,35 @@ static inline int io_remap_pfn_range(struct vm_area_struct *vma, /* We provide a special get_unmapped_area for framebuffer mmaps to try and use * the largest alignment possible such that larget PTEs can be used. */ -extern unsigned long get_fb_unmapped_area(struct file *filp, unsigned long, - unsigned long, unsigned long, - unsigned long); +unsigned long get_fb_unmapped_area(struct file *filp, unsigned long, + unsigned long, unsigned long, + unsigned long); #define HAVE_ARCH_FB_UNMAPPED_AREA -extern void pgtable_cache_init(void); -extern void sun4v_register_fault_status(void); -extern void sun4v_ktsb_register(void); -extern void __init cheetah_ecache_flush_init(void); -extern void sun4v_patch_tlb_handlers(void); +void sun4v_register_fault_status(void); +void sun4v_ktsb_register(void); +void __init cheetah_ecache_flush_init(void); +void sun4v_patch_tlb_handlers(void); extern unsigned long cmdline_memory_size; -extern asmlinkage void do_sparc64_fault(struct pt_regs *regs); +asmlinkage void do_sparc64_fault(struct pt_regs *regs); + +#define pmd_pgtable(PMD) ((pte_t *)pmd_page_vaddr(PMD)) + +#ifdef CONFIG_HUGETLB_PAGE + +#define pud_leaf_size pud_leaf_size +extern unsigned long pud_leaf_size(pud_t pud); + +#define pmd_leaf_size pmd_leaf_size +extern unsigned long pmd_leaf_size(pmd_t pmd); + +#define pte_leaf_size pte_leaf_size +extern unsigned long pte_leaf_size(pte_t pte); + +#endif /* CONFIG_HUGETLB_PAGE */ -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #endif /* !(_SPARC64_PGTABLE_H) */ diff --git a/arch/sparc/include/asm/pgtsrmmu.h b/arch/sparc/include/asm/pgtsrmmu.h index 79da17866fa8..a265822a475e 100644 --- a/arch/sparc/include/asm/pgtsrmmu.h +++ b/arch/sparc/include/asm/pgtsrmmu.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * pgtsrmmu.h: SRMMU page table defines and code. * @@ -9,46 +10,16 @@ #include <asm/page.h> -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #include <asm/thread_info.h> /* TI_UWINMASK for WINDOW_FLUSH */ #endif /* Number of contexts is implementation-dependent; 64k is the most we support */ #define SRMMU_MAX_CONTEXTS 65536 -/* PMD_SHIFT determines the size of the area a second-level page table entry can map */ -#define SRMMU_REAL_PMD_SHIFT 18 -#define SRMMU_REAL_PMD_SIZE (1UL << SRMMU_REAL_PMD_SHIFT) -#define SRMMU_REAL_PMD_MASK (~(SRMMU_REAL_PMD_SIZE-1)) -#define SRMMU_REAL_PMD_ALIGN(__addr) (((__addr)+SRMMU_REAL_PMD_SIZE-1)&SRMMU_REAL_PMD_MASK) - -/* PGDIR_SHIFT determines what a third-level page table entry can map */ -#define SRMMU_PGDIR_SHIFT 24 -#define SRMMU_PGDIR_SIZE (1UL << SRMMU_PGDIR_SHIFT) -#define SRMMU_PGDIR_MASK (~(SRMMU_PGDIR_SIZE-1)) -#define SRMMU_PGDIR_ALIGN(addr) (((addr)+SRMMU_PGDIR_SIZE-1)&SRMMU_PGDIR_MASK) - -#define SRMMU_REAL_PTRS_PER_PTE 64 -#define SRMMU_REAL_PTRS_PER_PMD 64 -#define SRMMU_PTRS_PER_PGD 256 - -#define SRMMU_REAL_PTE_TABLE_SIZE (SRMMU_REAL_PTRS_PER_PTE*4) -#define SRMMU_PMD_TABLE_SIZE (SRMMU_REAL_PTRS_PER_PMD*4) -#define SRMMU_PGD_TABLE_SIZE (SRMMU_PTRS_PER_PGD*4) - -/* - * To support pagetables in highmem, Linux introduces APIs which - * return struct page* and generally manipulate page tables when - * they are not mapped into kernel space. Our hardware page tables - * are smaller than pages. We lump hardware tabes into big, page sized - * software tables. - * - * PMD_SHIFT determines the size of the area a second-level page table entry - * can map, and our pmd_t is 16 times larger than normal. The values which - * were once defined here are now generic for 4c and srmmu, so they're - * found in pgtable.h. - */ -#define SRMMU_PTRS_PER_PMD 4 +#define SRMMU_PTE_TABLE_SIZE (PTRS_PER_PTE*4) +#define SRMMU_PMD_TABLE_SIZE (PTRS_PER_PMD*4) +#define SRMMU_PGD_TABLE_SIZE (PTRS_PER_PGD*4) /* Definition of the values in the ET field of PTD's and PTE's */ #define SRMMU_ET_MASK 0x3 @@ -80,27 +51,15 @@ #define SRMMU_PRIV 0x1c #define SRMMU_PRIV_RDONLY 0x18 -#define SRMMU_FILE 0x40 /* Implemented in software */ - -#define SRMMU_PTE_FILE_SHIFT 8 /* == 32-PTE_FILE_MAX_BITS */ - #define SRMMU_CHG_MASK (0xffffff00 | SRMMU_REF | SRMMU_DIRTY) -/* SRMMU swap entry encoding - * - * We use 5 bits for the type and 19 for the offset. This gives us - * 32 swapfiles of 4GB each. Encoding looks like: - * - * oooooooooooooooooootttttRRRRRRRR - * fedcba9876543210fedcba9876543210 - * - * The bottom 8 bits are reserved for protection and status bits, especially - * FILE and PRESENT. - */ +/* SRMMU swap entry encoding */ #define SRMMU_SWP_TYPE_MASK 0x1f -#define SRMMU_SWP_TYPE_SHIFT SRMMU_PTE_FILE_SHIFT -#define SRMMU_SWP_OFF_MASK 0x7ffff -#define SRMMU_SWP_OFF_SHIFT (SRMMU_PTE_FILE_SHIFT + 5) +#define SRMMU_SWP_TYPE_SHIFT 7 +#define SRMMU_SWP_OFF_MASK 0xfffff +#define SRMMU_SWP_OFF_SHIFT (SRMMU_SWP_TYPE_SHIFT + 5) +/* We borrow bit 6 to store the exclusive marker in swap PTEs. */ +#define SRMMU_SWP_EXCLUSIVE SRMMU_DIRTY /* Some day I will implement true fine grained access bits for * user pages because the SRMMU gives us the capabilities to @@ -138,7 +97,7 @@ bne 99b; \ restore %g0, %g0, %g0; -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ extern unsigned long last_valid_pfn; /* This makes sense. Honest it does - Anton */ @@ -146,7 +105,7 @@ extern unsigned long last_valid_pfn; extern void *srmmu_nocache_pool; #define __nocache_pa(VADDR) (((unsigned long)VADDR) - SRMMU_NOCACHE_VADDR + __pa((unsigned long)srmmu_nocache_pool)) #define __nocache_va(PADDR) (__va((unsigned long)PADDR) - (unsigned long)srmmu_nocache_pool + SRMMU_NOCACHE_VADDR) -#define __nocache_fix(VADDR) __va(__nocache_pa(VADDR)) +#define __nocache_fix(VADDR) ((__typeof__(VADDR))__va(__nocache_pa(VADDR))) /* Accessing the MMU control register. */ unsigned int srmmu_get_mmureg(void); @@ -177,6 +136,6 @@ srmmu_get_pte (unsigned long addr) return entry; } -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #endif /* !(_SPARC_PGTSRMMU_H) */ diff --git a/arch/sparc/include/asm/pil.h b/arch/sparc/include/asm/pil.h index 266937030546..4003c35304bd 100644 --- a/arch/sparc/include/asm/pil.h +++ b/arch/sparc/include/asm/pil.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_PIL_H #define _SPARC64_PIL_H @@ -20,7 +21,6 @@ #define PIL_SMP_CALL_FUNC 1 #define PIL_SMP_RECEIVE_SIGNAL 2 #define PIL_SMP_CAPTURE 3 -#define PIL_SMP_CTX_NEW_VERSION 4 #define PIL_DEVICE_IRQ 5 #define PIL_SMP_CALL_FUNC_SNGL 6 #define PIL_DEFERRED_PCR_WORK 7 diff --git a/arch/sparc/include/asm/processor.h b/arch/sparc/include/asm/processor.h index 2fe99e66e760..18295ea625dd 100644 --- a/arch/sparc/include/asm/processor.h +++ b/arch/sparc/include/asm/processor.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_PROCESSOR_H #define ___ASM_SPARC_PROCESSOR_H #if defined(__sparc__) && defined(__arch64__) @@ -5,7 +6,4 @@ #else #include <asm/processor_32.h> #endif - -#define nop() __asm__ __volatile__ ("nop") - #endif diff --git a/arch/sparc/include/asm/processor_32.h b/arch/sparc/include/asm/processor_32.h index 2c7baa4c4505..ba8b70ffec08 100644 --- a/arch/sparc/include/asm/processor_32.h +++ b/arch/sparc/include/asm/processor_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* include/asm/processor.h * * Copyright (C) 1994 David S. Miller (davem@caip.rutgers.edu) @@ -6,24 +7,12 @@ #ifndef __ASM_SPARC_PROCESSOR_H #define __ASM_SPARC_PROCESSOR_H -/* - * Sparc32 implementation of macro that returns current - * instruction pointer ("program counter"). - */ -#define current_text_addr() ({ void *pc; __asm__("sethi %%hi(1f), %0; or %0, %%lo(1f), %0;\n1:" : "=r" (pc)); pc; }) - #include <asm/psr.h> #include <asm/ptrace.h> #include <asm/head.h> #include <asm/signal.h> #include <asm/page.h> -/* - * The sparc has no problems with write protection - */ -#define wp_works_ok 1 -#define wp_works_ok__is_a_macro /* for versions in ksyms.c */ - /* Whee, this is STACK_TOP + PAGE_SIZE and the lowest kernel address too... * That one page is used to protect kernel from intruders, so that * we can make our access_ok test faster @@ -43,10 +32,6 @@ struct fpq { }; #endif -typedef struct { - int seg; -} mm_segment_t; - /* The Sparc processor specific thread struct. */ struct thread_struct { struct pt_regs *kregs; @@ -61,21 +46,12 @@ struct thread_struct { unsigned long fsr; unsigned long fpqdepth; struct fpq fpqueue[16]; - unsigned long flags; - mm_segment_t current_ds; }; -#define SPARC_FLAG_KTHREAD 0x1 /* task is a kernel thread */ -#define SPARC_FLAG_UNALIGNED 0x2 /* is allowed to do unaligned accesses */ - #define INIT_THREAD { \ - .flags = SPARC_FLAG_KTHREAD, \ - .current_ds = KERNEL_DS, \ + .kregs = (struct pt_regs *)(init_stack+THREAD_SIZE)-1 \ } -/* Return saved PC of a blocked thread. */ -extern unsigned long thread_saved_pc(struct task_struct *t); - /* Do necessary setup to start up a newly executed thread. */ static inline void start_thread(struct pt_regs * regs, unsigned long pc, unsigned long sp) @@ -104,10 +80,7 @@ static inline void start_thread(struct pt_regs * regs, unsigned long pc, : "memory"); } -/* Free all resources held by a thread. */ -#define release_thread(tsk) do { } while(0) - -extern unsigned long get_wchan(struct task_struct *); +unsigned long __get_wchan(struct task_struct *); #define task_pt_regs(tsk) ((tsk)->thread.kregs) #define KSTK_EIP(tsk) ((tsk)->thread.kregs->pc) @@ -116,8 +89,10 @@ extern unsigned long get_wchan(struct task_struct *); #ifdef __KERNEL__ extern struct task_struct *last_task_used_math; +int do_mathemu(struct pt_regs *regs, struct task_struct *fpt); #define cpu_relax() barrier() + extern void (*sparc_idle)(void); #endif diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h index 4c3f7f01c709..321859454ca4 100644 --- a/arch/sparc/include/asm/processor_64.h +++ b/arch/sparc/include/asm/processor_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * include/asm/processor.h * @@ -7,21 +8,11 @@ #ifndef __ASM_SPARC64_PROCESSOR_H #define __ASM_SPARC64_PROCESSOR_H -/* - * Sparc64 implementation of macro that returns current - * instruction pointer ("program counter"). - */ -#define current_text_addr() ({ void *pc; __asm__("rd %%pc, %0" : "=r" (pc)); pc; }) - #include <asm/asi.h> #include <asm/pstate.h> #include <asm/ptrace.h> #include <asm/page.h> -/* The sparc has no problems with write protection */ -#define wp_works_ok 1 -#define wp_works_ok__is_a_macro /* for versions in ksyms.c */ - /* * User lives in his very own context, and cannot reference us. Note * that TASK_SIZE is a misnomer, it really gives maximum user virtual @@ -30,7 +21,7 @@ * XXX No longer using virtual page tables, kill this upper limit... */ #define VA_BITS 44 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define VPTE_SIZE (1UL << (VA_BITS - PAGE_SHIFT + 3)) #else #define VPTE_SIZE (1 << (VA_BITS - PAGE_SHIFT + 3)) @@ -54,11 +45,7 @@ #endif -#ifndef __ASSEMBLY__ - -typedef struct { - unsigned char seg; -} mm_segment_t; +#ifndef __ASSEMBLER__ /* The Sparc processor specific thread struct. */ /* XXX This should die, everything can go into thread_info now. */ @@ -75,7 +62,7 @@ struct thread_struct { #endif }; -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #ifndef CONFIG_DEBUG_SPINLOCK #define INIT_THREAD { \ @@ -88,14 +75,12 @@ struct thread_struct { } #endif /* !(CONFIG_DEBUG_SPINLOCK) */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> #include <asm/fpumacro.h> -/* Return saved PC of a blocked thread. */ struct task_struct; -extern unsigned long thread_saved_pc(struct task_struct *); /* On Uniprocessor, even in RMO processes see TSO semantics */ #ifdef CONFIG_SMP @@ -191,20 +176,24 @@ do { \ regs->tstate &= ~TSTATE_PEF; \ } while (0) -/* Free all resources held by a thread. */ -#define release_thread(tsk) do { } while (0) - -extern unsigned long get_wchan(struct task_struct *task); +unsigned long __get_wchan(struct task_struct *task); #define task_pt_regs(tsk) (task_thread_info(tsk)->kregs) #define KSTK_EIP(tsk) (task_pt_regs(tsk)->tpc) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->u_regs[UREG_FP]) /* Please see the commentary in asm/backoff.h for a description of - * what these instructions are doing and how they have been choosen. + * what these instructions are doing and how they have been chosen. * To make a long story short, we are trying to yield the current cpu * strand during busy loops. */ +#ifdef BUILD_VDSO +#define cpu_relax() asm volatile("\n99:\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + ::: "memory") +#else /* ! BUILD_VDSO */ #define cpu_relax() asm volatile("\n99:\n\t" \ "rd %%ccr, %%g0\n\t" \ "rd %%ccr, %%g0\n\t" \ @@ -216,6 +205,7 @@ extern unsigned long get_wchan(struct task_struct *task); "nop\n\t" \ ".previous" \ ::: "memory") +#endif /* Prefetch support. This is tuned for UltraSPARC-III and later. * UltraSPARC-I will treat these as nops, and UltraSPARC-II has @@ -223,7 +213,6 @@ extern unsigned long get_wchan(struct task_struct *task); */ #define ARCH_HAS_PREFETCH #define ARCH_HAS_PREFETCHW -#define ARCH_HAS_SPINLOCK_PREFETCH static inline void prefetch(const void *x) { @@ -249,10 +238,10 @@ static inline void prefetchw(const void *x) : "r" (x)); } -#define spin_lock_prefetch(x) prefetchw(x) - #define HAVE_ARCH_PICK_MMAP_LAYOUT -#endif /* !(__ASSEMBLY__) */ +int do_mathemu(struct pt_regs *regs, struct fpustate *f, bool illegal_insn_trap); + +#endif /* !(__ASSEMBLER__) */ #endif /* !(__ASM_SPARC64_PROCESSOR_H) */ diff --git a/arch/sparc/include/asm/prom.h b/arch/sparc/include/asm/prom.h index 67c62578d170..8184575b1336 100644 --- a/arch/sparc/include/asm/prom.h +++ b/arch/sparc/include/asm/prom.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ #include <linux/of.h> /* linux/of.h gets to determine #include ordering */ #ifndef _SPARC_PROM_H #define _SPARC_PROM_H @@ -11,11 +12,6 @@ * * Updates for PPC64 by Peter Bergner & David Engebretsen, IBM Corp. * Updates for SPARC by David S. Miller - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #include <linux/types.h> #include <linux/of_pdt.h> @@ -23,53 +19,41 @@ #include <linux/mutex.h> #include <linux/atomic.h> #include <linux/irqdomain.h> - -#define OF_ROOT_NODE_ADDR_CELLS_DEFAULT 2 -#define OF_ROOT_NODE_SIZE_CELLS_DEFAULT 1 +#include <linux/spinlock.h> #define of_compat_cmp(s1, s2, l) strncmp((s1), (s2), (l)) #define of_prop_cmp(s1, s2) strcasecmp((s1), (s2)) #define of_node_cmp(s1, s2) strcmp((s1), (s2)) +extern raw_spinlock_t devtree_lock; + struct of_irq_controller { unsigned int (*irq_build)(struct device_node *, unsigned int, void *); void *data; }; -extern struct device_node *of_find_node_by_cpuid(int cpuid); -extern int of_set_property(struct device_node *node, const char *name, void *val, int len); +struct device_node *of_find_node_by_cpuid(int cpuid); +int of_set_property(struct device_node *node, const char *name, void *val, int len); extern struct mutex of_set_property_mutex; -extern int of_getintprop_default(struct device_node *np, - const char *name, +int of_getintprop_default(struct device_node *np, + const char *name, int def); -extern int of_find_in_proplist(const char *list, const char *match, int len); -#ifdef CONFIG_NUMA -extern int of_node_to_nid(struct device_node *dp); -#define of_node_to_nid of_node_to_nid -#endif +int of_find_in_proplist(const char *list, const char *match, int len); -extern void prom_build_devicetree(void); -extern void of_populate_present_mask(void); -extern void of_fill_in_cpu_data(void); +void prom_build_devicetree(void); +void of_populate_present_mask(void); +void of_fill_in_cpu_data(void); struct resource; -extern void __iomem *of_ioremap(struct resource *res, unsigned long offset, unsigned long size, char *name); -extern void of_iounmap(struct resource *res, void __iomem *base, unsigned long size); +void __iomem *of_ioremap(struct resource *res, unsigned long offset, unsigned long size, char *name); +void of_iounmap(struct resource *res, void __iomem *base, unsigned long size); extern struct device_node *of_console_device; extern char *of_console_path; extern char *of_console_options; -extern void irq_trans_init(struct device_node *dp); -extern char *build_path_component(struct device_node *dp); - -/* SPARC has local implementations */ -extern int of_address_to_resource(struct device_node *dev, int index, - struct resource *r); -#define of_address_to_resource of_address_to_resource - -void __iomem *of_iomap(struct device_node *node, int index); -#define of_iomap of_iomap +void irq_trans_init(struct device_node *dp); +char *build_path_component(struct device_node *dp); #endif /* __KERNEL__ */ #endif /* _SPARC_PROM_H */ diff --git a/arch/sparc/include/asm/psr.h b/arch/sparc/include/asm/psr.h index e71eb57945e0..5af50ccda023 100644 --- a/arch/sparc/include/asm/psr.h +++ b/arch/sparc/include/asm/psr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * psr.h: This file holds the macros for masking off various parts of * the processor status register on the Sparc. This is valid @@ -13,7 +14,7 @@ #include <uapi/asm/psr.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* Get the %psr register. */ static inline unsigned int get_psr(void) { @@ -62,6 +63,6 @@ static inline unsigned int get_fsr(void) return fsr; } -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #endif /* !(__LINUX_SPARC_PSR_H) */ diff --git a/arch/sparc/include/asm/ptrace.h b/arch/sparc/include/asm/ptrace.h index bdfafd7af46f..8adf3fd2f00f 100644 --- a/arch/sparc/include/asm/ptrace.h +++ b/arch/sparc/include/asm/ptrace.h @@ -1,11 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_PTRACE_H #define __SPARC_PTRACE_H #include <uapi/asm/ptrace.h> #if defined(__sparc__) && defined(__arch64__) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ +#include <linux/compiler.h> #include <linux/threads.h> #include <asm/switch_to.h> @@ -24,12 +26,12 @@ static inline bool pt_regs_clear_syscall(struct pt_regs *regs) return (regs->tstate &= ~TSTATE_SYSCALL); } -#define arch_ptrace_stop_needed(exit_code, info) \ +#define arch_ptrace_stop_needed() \ ({ flush_user_windows(); \ get_thread_wsaved() != 0; \ }) -#define arch_ptrace_stop(exit_code, info) \ +#define arch_ptrace_stop() \ synchronize_user_stack() #define current_pt_regs() \ @@ -61,7 +63,10 @@ extern union global_cpu_snapshot global_cpu_snapshot[NR_CPUS]; #define force_successful_syscall_return() set_thread_noerror(1) #define user_mode(regs) (!((regs)->tstate & TSTATE_PRIV)) #define instruction_pointer(regs) ((regs)->tpc) -#define instruction_pointer_set(regs, val) ((regs)->tpc = (val)) +#define instruction_pointer_set(regs, val) do { \ + (regs)->tpc = (val); \ + (regs)->tnpc = (val)+4; \ + } while (0) #define user_stack_pointer(regs) ((regs)->u_regs[UREG_FP]) static inline int is_syscall_success(struct pt_regs *regs) { @@ -73,14 +78,45 @@ static inline long regs_return_value(struct pt_regs *regs) return regs->u_regs[UREG_I0]; } #ifdef CONFIG_SMP -extern unsigned long profile_pc(struct pt_regs *); +unsigned long profile_pc(struct pt_regs *); #else #define profile_pc(regs) instruction_pointer(regs) #endif -#else /* __ASSEMBLY__ */ -#endif /* __ASSEMBLY__ */ + +#define MAX_REG_OFFSET (offsetof(struct pt_regs, magic)) + +int regs_query_register_offset(const char *name); +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n); + +/** + * regs_get_register() - get register value from its offset + * @regs: pt_regs from which register value is gotten + * @offset: offset number of the register. + * + * regs_get_register returns the value of a register whose + * offset from @regs. The @offset is the offset of the register + * in struct pt_regs. If @offset is bigger than MAX_REG_OFFSET, + * this returns 0. + */ +static inline unsigned long regs_get_register(struct pt_regs *regs, + unsigned long offset) +{ + if (unlikely(offset >= MAX_REG_OFFSET)) + return 0; + if (offset == PT_V9_Y) + return *(unsigned int *)((unsigned long)regs + offset); + return *(unsigned long *)((unsigned long)regs + offset); +} + +/* Valid only for Kernel mode traps. */ +static inline unsigned long kernel_stack_pointer(struct pt_regs *regs) +{ + return regs->u_regs[UREG_I6]; +} +#else /* __ASSEMBLER__ */ +#endif /* __ASSEMBLER__ */ #else /* (defined(__sparc__) && defined(__arch64__)) */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/switch_to.h> static inline bool pt_regs_is_syscall(struct pt_regs *regs) @@ -93,12 +129,12 @@ static inline bool pt_regs_clear_syscall(struct pt_regs *regs) return (regs->psr &= ~PSR_SYSCALL); } -#define arch_ptrace_stop_needed(exit_code, info) \ +#define arch_ptrace_stop_needed() \ ({ flush_user_windows(); \ current_thread_info()->w_saved != 0; \ }) -#define arch_ptrace_stop(exit_code, info) \ +#define arch_ptrace_stop() \ synchronize_user_stack() #define current_pt_regs() \ @@ -108,8 +144,8 @@ static inline bool pt_regs_clear_syscall(struct pt_regs *regs) #define instruction_pointer(regs) ((regs)->pc) #define user_stack_pointer(regs) ((regs)->u_regs[UREG_FP]) unsigned long profile_pc(struct pt_regs *); -#else /* (!__ASSEMBLY__) */ -#endif /* (!__ASSEMBLY__) */ +#else /* (!__ASSEMBLER__) */ +#endif /* (!__ASSEMBLER__) */ #endif /* (defined(__sparc__) && defined(__arch64__)) */ #define STACK_BIAS 2047 diff --git a/arch/sparc/include/asm/qrwlock.h b/arch/sparc/include/asm/qrwlock.h new file mode 100644 index 000000000000..c277729152c6 --- /dev/null +++ b/arch/sparc/include/asm/qrwlock.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SPARC_QRWLOCK_H +#define _ASM_SPARC_QRWLOCK_H + +#include <asm-generic/qrwlock_types.h> +#include <asm-generic/qrwlock.h> + +#endif /* _ASM_SPARC_QRWLOCK_H */ diff --git a/arch/sparc/include/asm/qspinlock.h b/arch/sparc/include/asm/qspinlock.h new file mode 100644 index 000000000000..48808f3d9dd7 --- /dev/null +++ b/arch/sparc/include/asm/qspinlock.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SPARC_QSPINLOCK_H +#define _ASM_SPARC_QSPINLOCK_H + +#include <asm-generic/qspinlock_types.h> +#include <asm-generic/qspinlock.h> + +#endif /* _ASM_SPARC_QSPINLOCK_H */ diff --git a/arch/sparc/include/asm/ross.h b/arch/sparc/include/asm/ross.h index ecb6e81786a6..53a42b37495d 100644 --- a/arch/sparc/include/asm/ross.h +++ b/arch/sparc/include/asm/ross.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * ross.h: Ross module specific definitions and defines. * @@ -94,7 +95,7 @@ #define HYPERSPARC_ICCR_FTD 0x00000002 #define HYPERSPARC_ICCR_ICE 0x00000001 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ static inline unsigned int get_ross_icr(void) { @@ -186,6 +187,6 @@ static inline void hyper_flush_cache_page(unsigned long page) } } -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #endif /* !(_SPARC_ROSS_H) */ diff --git a/arch/sparc/include/asm/rwsem.h b/arch/sparc/include/asm/rwsem.h deleted file mode 100644 index 069bf4d663a1..000000000000 --- a/arch/sparc/include/asm/rwsem.h +++ /dev/null @@ -1,124 +0,0 @@ -/* - * rwsem.h: R/W semaphores implemented using CAS - * - * Written by David S. Miller (davem@redhat.com), 2001. - * Derived from asm-i386/rwsem.h - */ -#ifndef _SPARC64_RWSEM_H -#define _SPARC64_RWSEM_H - -#ifndef _LINUX_RWSEM_H -#error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead" -#endif - -#ifdef __KERNEL__ - -#define RWSEM_UNLOCKED_VALUE 0x00000000L -#define RWSEM_ACTIVE_BIAS 0x00000001L -#define RWSEM_ACTIVE_MASK 0xffffffffL -#define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1) -#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS -#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) - -/* - * lock for reading - */ -static inline void __down_read(struct rw_semaphore *sem) -{ - if (unlikely(atomic64_inc_return((atomic64_t *)(&sem->count)) <= 0L)) - rwsem_down_read_failed(sem); -} - -static inline int __down_read_trylock(struct rw_semaphore *sem) -{ - long tmp; - - while ((tmp = sem->count) >= 0L) { - if (tmp == cmpxchg(&sem->count, tmp, - tmp + RWSEM_ACTIVE_READ_BIAS)) { - return 1; - } - } - return 0; -} - -/* - * lock for writing - */ -static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) -{ - long tmp; - - tmp = atomic64_add_return(RWSEM_ACTIVE_WRITE_BIAS, - (atomic64_t *)(&sem->count)); - if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS)) - rwsem_down_write_failed(sem); -} - -static inline void __down_write(struct rw_semaphore *sem) -{ - __down_write_nested(sem, 0); -} - -static inline int __down_write_trylock(struct rw_semaphore *sem) -{ - long tmp; - - tmp = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE, - RWSEM_ACTIVE_WRITE_BIAS); - return tmp == RWSEM_UNLOCKED_VALUE; -} - -/* - * unlock after reading - */ -static inline void __up_read(struct rw_semaphore *sem) -{ - long tmp; - - tmp = atomic64_dec_return((atomic64_t *)(&sem->count)); - if (unlikely(tmp < -1L && (tmp & RWSEM_ACTIVE_MASK) == 0L)) - rwsem_wake(sem); -} - -/* - * unlock after writing - */ -static inline void __up_write(struct rw_semaphore *sem) -{ - if (unlikely(atomic64_sub_return(RWSEM_ACTIVE_WRITE_BIAS, - (atomic64_t *)(&sem->count)) < 0L)) - rwsem_wake(sem); -} - -/* - * implement atomic add functionality - */ -static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem) -{ - atomic64_add(delta, (atomic64_t *)(&sem->count)); -} - -/* - * downgrade write lock to read lock - */ -static inline void __downgrade_write(struct rw_semaphore *sem) -{ - long tmp; - - tmp = atomic64_add_return(-RWSEM_WAITING_BIAS, (atomic64_t *)(&sem->count)); - if (tmp < 0L) - rwsem_downgrade_wake(sem); -} - -/* - * implement exchange and add functionality - */ -static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) -{ - return atomic64_add_return(delta, (atomic64_t *)(&sem->count)); -} - -#endif /* __KERNEL__ */ - -#endif /* _SPARC64_RWSEM_H */ diff --git a/arch/sparc/include/asm/sbi.h b/arch/sparc/include/asm/sbi.h index 5eb7f1965d33..861f85b5bf9b 100644 --- a/arch/sparc/include/asm/sbi.h +++ b/arch/sparc/include/asm/sbi.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * sbi.h: SBI (Sbus Interface on sun4d) definitions * @@ -63,7 +64,7 @@ struct sbi_regs { */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ static inline int acquire_sbi(int devid, int mask) { @@ -110,6 +111,6 @@ static inline void set_sbi_ctl(int devid, int cfgno, int cfg) "i" (ASI_M_CTL)); } -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* !(_SPARC_SBI_H) */ diff --git a/arch/sparc/include/asm/scatterlist.h b/arch/sparc/include/asm/scatterlist.h deleted file mode 100644 index 92bb638313f8..000000000000 --- a/arch/sparc/include/asm/scatterlist.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef _SPARC_SCATTERLIST_H -#define _SPARC_SCATTERLIST_H - -#include <asm-generic/scatterlist.h> - -#define ARCH_HAS_SG_CHAIN - -#endif /* !(_SPARC_SCATTERLIST_H) */ diff --git a/arch/sparc/include/asm/scratchpad.h b/arch/sparc/include/asm/scratchpad.h index 5e8b01fb3343..958351190788 100644 --- a/arch/sparc/include/asm/scratchpad.h +++ b/arch/sparc/include/asm/scratchpad.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_SCRATCHPAD_H #define _SPARC64_SCRATCHPAD_H diff --git a/arch/sparc/include/asm/seccomp.h b/arch/sparc/include/asm/seccomp.h index adca1bce41d4..62d4579efb1a 100644 --- a/arch/sparc/include/asm/seccomp.h +++ b/arch/sparc/include/asm/seccomp.h @@ -1,15 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_SECCOMP_H +#define _ASM_SECCOMP_H #include <linux/unistd.h> -#define __NR_seccomp_read __NR_read -#define __NR_seccomp_write __NR_write -#define __NR_seccomp_exit __NR_exit -#define __NR_seccomp_sigreturn __NR_rt_sigreturn - -#define __NR_seccomp_read_32 __NR_read -#define __NR_seccomp_write_32 __NR_write -#define __NR_seccomp_exit_32 __NR_exit #define __NR_seccomp_sigreturn_32 __NR_sigreturn +#include <asm-generic/seccomp.h> + #endif /* _ASM_SECCOMP_H */ diff --git a/arch/sparc/include/asm/sections.h b/arch/sparc/include/asm/sections.h index f300d1a9b2b6..08f833453ab3 100644 --- a/arch/sparc/include/asm/sections.h +++ b/arch/sparc/include/asm/sections.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_SECTIONS_H #define __SPARC_SECTIONS_H diff --git a/arch/sparc/include/asm/setup.h b/arch/sparc/include/asm/setup.h index 5e35e0517318..72205684e51e 100644 --- a/arch/sparc/include/asm/setup.h +++ b/arch/sparc/include/asm/setup.h @@ -1,11 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* - * Just a place holder. + * Just a place holder. */ #ifndef _SPARC_SETUP_H #define _SPARC_SETUP_H -#include <uapi/asm/setup.h> +#include <linux/interrupt.h> +#include <uapi/asm/setup.h> extern char reboot_command[]; @@ -15,16 +17,55 @@ extern char reboot_command[]; */ extern unsigned char boot_cpu_id; -extern unsigned long empty_zero_page; +extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; extern int serial_console; static inline int con_is_present(void) { return serial_console ? 0 : 1; } + +/* from irq_32.c */ +extern volatile unsigned char *fdc_status; +extern char *pdma_vaddr; +extern unsigned long pdma_size; +extern volatile int doing_pdma; + +/* This is software state */ +extern char *pdma_base; +extern unsigned long pdma_areasize; + +int sparc_floppy_request_irq(unsigned int irq, irq_handler_t irq_handler); + +/* setup_32.c */ +extern unsigned long cmdline_memory_size; + +/* devices.c */ +void __init device_scan(void); + +/* unaligned_32.c */ +unsigned long safe_compute_effective_address(struct pt_regs *, unsigned int); + #endif -extern void sun_do_break(void); +#ifdef CONFIG_SPARC64 +void __init start_early_boot(void); + +/* unaligned_64.c */ +int handle_ldf_stq(u32 insn, struct pt_regs *regs); +void handle_ld_nf(u32 insn, struct pt_regs *regs); + +/* init_64.c */ +extern atomic_t dcpage_flushes; +extern atomic_t dcpage_flushes_xcall; + +extern int sysctl_tsb_ratio; + +#ifdef CONFIG_SERIAL_SUNHV +void sunhv_migrate_hvcons_irq(int cpu); +#endif +#endif +void sun_do_break(void); extern int stop_a_enabled; extern int scons_pwroff; diff --git a/arch/sparc/include/asm/sfafsr.h b/arch/sparc/include/asm/sfafsr.h index e96137b04a4f..9c98f4fc8839 100644 --- a/arch/sparc/include/asm/sfafsr.h +++ b/arch/sparc/include/asm/sfafsr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_SFAFSR_H #define _SPARC64_SFAFSR_H diff --git a/arch/sparc/include/asm/sfp-machine.h b/arch/sparc/include/asm/sfp-machine.h index 4ebc3823ed4f..4a247b40573c 100644 --- a/arch/sparc/include/asm/sfp-machine.h +++ b/arch/sparc/include/asm/sfp-machine.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_SFP_MACHINE_H #define ___ASM_SPARC_SFP_MACHINE_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/sfp-machine_32.h b/arch/sparc/include/asm/sfp-machine_32.h index 01d9c3b5a73b..838c9d58f3b4 100644 --- a/arch/sparc/include/asm/sfp-machine_32.h +++ b/arch/sparc/include/asm/sfp-machine_32.h @@ -79,9 +79,9 @@ __asm__ ("addcc %r7,%8,%2\n\t" \ "addxcc %r5,%6,%1\n\t" \ "addx %r3,%4,%0\n" \ - : "=r" ((USItype)(r2)), \ - "=&r" ((USItype)(r1)), \ - "=&r" ((USItype)(r0)) \ + : "=r" (r2), \ + "=&r" (r1), \ + "=&r" (r0) \ : "%rJ" ((USItype)(x2)), \ "rI" ((USItype)(y2)), \ "%rJ" ((USItype)(x1)), \ @@ -94,9 +94,9 @@ __asm__ ("subcc %r7,%8,%2\n\t" \ "subxcc %r5,%6,%1\n\t" \ "subx %r3,%4,%0\n" \ - : "=r" ((USItype)(r2)), \ - "=&r" ((USItype)(r1)), \ - "=&r" ((USItype)(r0)) \ + : "=r" (r2), \ + "=&r" (r1), \ + "=&r" (r0) \ : "%rJ" ((USItype)(x2)), \ "rI" ((USItype)(y2)), \ "%rJ" ((USItype)(x1)), \ @@ -115,8 +115,8 @@ "addxcc %r6,%7,%0\n\t" \ "addxcc %r4,%5,%%g2\n\t" \ "addx %r2,%3,%%g1\n\t" \ - : "=&r" ((USItype)(r1)), \ - "=&r" ((USItype)(r0)) \ + : "=&r" (r1), \ + "=&r" (r0) \ : "%rJ" ((USItype)(x3)), \ "rI" ((USItype)(y3)), \ "%rJ" ((USItype)(x2)), \ @@ -140,8 +140,8 @@ "subxcc %r6,%7,%0\n\t" \ "subxcc %r4,%5,%%g2\n\t" \ "subx %r2,%3,%%g1\n\t" \ - : "=&r" ((USItype)(r1)), \ - "=&r" ((USItype)(r0)) \ + : "=&r" (r1), \ + "=&r" (r0) \ : "%rJ" ((USItype)(x3)), \ "rI" ((USItype)(y3)), \ "%rJ" ((USItype)(x2)), \ @@ -164,10 +164,10 @@ "addxcc %2,%%g0,%2\n\t" \ "addxcc %1,%%g0,%1\n\t" \ "addx %0,%%g0,%0\n\t" \ - : "=&r" ((USItype)(x3)), \ - "=&r" ((USItype)(x2)), \ - "=&r" ((USItype)(x1)), \ - "=&r" ((USItype)(x0)) \ + : "=&r" (x3), \ + "=&r" (x2), \ + "=&r" (x1), \ + "=&r" (x0) \ : "rI" ((USItype)(i)), \ "0" ((USItype)(x3)), \ "1" ((USItype)(x2)), \ diff --git a/arch/sparc/include/asm/shmparam.h b/arch/sparc/include/asm/shmparam.h index 8bf0cfe0694f..951a4525fa4b 100644 --- a/arch/sparc/include/asm/shmparam.h +++ b/arch/sparc/include/asm/shmparam.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_SHMPARAM_H #define ___ASM_SPARC_SHMPARAM_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/shmparam_32.h b/arch/sparc/include/asm/shmparam_32.h index 142825c8d3ac..9767a8b30242 100644 --- a/arch/sparc/include/asm/shmparam_32.h +++ b/arch/sparc/include/asm/shmparam_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASMSPARC_SHMPARAM_H #define _ASMSPARC_SHMPARAM_H diff --git a/arch/sparc/include/asm/shmparam_64.h b/arch/sparc/include/asm/shmparam_64.h index 1ed0d6701a9b..c0731b57124c 100644 --- a/arch/sparc/include/asm/shmparam_64.h +++ b/arch/sparc/include/asm/shmparam_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASMSPARC64_SHMPARAM_H #define _ASMSPARC64_SHMPARAM_H diff --git a/arch/sparc/include/asm/sigcontext.h b/arch/sparc/include/asm/sigcontext.h index fc2df1e892cb..200f95144fd2 100644 --- a/arch/sparc/include/asm/sigcontext.h +++ b/arch/sparc/include/asm/sigcontext.h @@ -1,10 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_SIGCONTEXT_H #define __SPARC_SIGCONTEXT_H #include <asm/ptrace.h> #include <uapi/asm/sigcontext.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define __SUNOS_MAXWIN 31 @@ -25,7 +26,7 @@ struct sigcontext32 { int sigc_oswins; /* outstanding windows */ /* stack ptrs for each regwin buf */ - unsigned sigc_spbuf[__SUNOS_MAXWIN]; + unsigned int sigc_spbuf[__SUNOS_MAXWIN]; /* Windows to restore after signal */ struct reg_window32 sigc_wbuf[__SUNOS_MAXWIN]; @@ -103,6 +104,6 @@ typedef struct { #endif /* (CONFIG_SPARC64) */ -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #endif /* !(__SPARC_SIGCONTEXT_H) */ diff --git a/arch/sparc/include/asm/siginfo.h b/arch/sparc/include/asm/siginfo.h deleted file mode 100644 index 48c34c19f810..000000000000 --- a/arch/sparc/include/asm/siginfo.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef __SPARC_SIGINFO_H -#define __SPARC_SIGINFO_H - -#include <uapi/asm/siginfo.h> - - -#ifdef CONFIG_COMPAT - -struct compat_siginfo; - -#endif /* CONFIG_COMPAT */ - -#endif /* !(__SPARC_SIGINFO_H) */ diff --git a/arch/sparc/include/asm/signal.h b/arch/sparc/include/asm/signal.h index c33ce3f2ba84..d93fe93544ec 100644 --- a/arch/sparc/include/asm/signal.h +++ b/arch/sparc/include/asm/signal.h @@ -1,28 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_SIGNAL_H #define __SPARC_SIGNAL_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/personality.h> #include <linux/types.h> #endif #include <uapi/asm/signal.h> -#ifndef __ASSEMBLY__ -/* - * DJHR - * SA_STATIC_ALLOC is used for the sparc32 system to indicate that this - * interrupt handler's irq structure should be statically allocated - * by the request_irq routine. - * The alternative is that arch/sparc/kernel/irq.c has carnal knowledge - * of interrupt usage and that sucks. Also without a flag like this - * it may be possible for the free_irq routine to attempt to free - * statically allocated data.. which is NOT GOOD. - * - */ -#define SA_STATIC_ALLOC 0x8000 +#ifndef __ASSEMBLER__ #define __ARCH_HAS_KA_RESTORER #define __ARCH_HAS_SA_RESTORER -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #endif /* !(__SPARC_SIGNAL_H) */ diff --git a/arch/sparc/include/asm/smp.h b/arch/sparc/include/asm/smp.h index b59672d0e19b..dea59f6ce79f 100644 --- a/arch/sparc/include/asm/smp.h +++ b/arch/sparc/include/asm/smp.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_SMP_H #define ___ASM_SPARC_SMP_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/smp_32.h b/arch/sparc/include/asm/smp_32.h index 3c8917f054de..9c6ed98fbaf1 100644 --- a/arch/sparc/include/asm/smp_32.h +++ b/arch/sparc/include/asm/smp_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* smp.h: Sparc specific SMP stuff. * * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) @@ -9,15 +10,15 @@ #include <linux/threads.h> #include <asm/head.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/cpumask.h> -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #ifdef CONFIG_SMP -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/ptrace.h> #include <asm/asi.h> @@ -32,9 +33,6 @@ extern volatile unsigned long cpu_callin_map[NR_CPUS]; extern cpumask_t smp_commenced_mask; extern struct linux_prom_registers smp_penguin_ctable; -typedef void (*smpfunc_t)(unsigned long, unsigned long, unsigned long, - unsigned long, unsigned long); - void cpu_panic(void); /* @@ -56,7 +54,7 @@ void smp_bogo(struct seq_file *); void smp_info(struct seq_file *); struct sparc32_ipi_ops { - void (*cross_call)(smpfunc_t func, cpumask_t mask, unsigned long arg1, + void (*cross_call)(void *func, cpumask_t mask, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4); void (*resched)(int cpu); @@ -65,49 +63,49 @@ struct sparc32_ipi_ops { }; extern const struct sparc32_ipi_ops *sparc32_ipi_ops; -static inline void xc0(smpfunc_t func) +static inline void xc0(void *func) { sparc32_ipi_ops->cross_call(func, *cpu_online_mask, 0, 0, 0, 0); } -static inline void xc1(smpfunc_t func, unsigned long arg1) +static inline void xc1(void *func, unsigned long arg1) { sparc32_ipi_ops->cross_call(func, *cpu_online_mask, arg1, 0, 0, 0); } -static inline void xc2(smpfunc_t func, unsigned long arg1, unsigned long arg2) +static inline void xc2(void *func, unsigned long arg1, unsigned long arg2) { sparc32_ipi_ops->cross_call(func, *cpu_online_mask, arg1, arg2, 0, 0); } -static inline void xc3(smpfunc_t func, unsigned long arg1, unsigned long arg2, +static inline void xc3(void *func, unsigned long arg1, unsigned long arg2, unsigned long arg3) { sparc32_ipi_ops->cross_call(func, *cpu_online_mask, arg1, arg2, arg3, 0); } -static inline void xc4(smpfunc_t func, unsigned long arg1, unsigned long arg2, +static inline void xc4(void *func, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4) { sparc32_ipi_ops->cross_call(func, *cpu_online_mask, arg1, arg2, arg3, arg4); } -extern void arch_send_call_function_single_ipi(int cpu); -extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); +void arch_send_call_function_single_ipi(int cpu); +void arch_send_call_function_ipi_mask(const struct cpumask *mask); static inline int cpu_logical_map(int cpu) { return cpu; } -extern int hard_smp_processor_id(void); +int hard_smp_processor_id(void); #define raw_smp_processor_id() (current_thread_info()->cpu) void smp_setup_cpu_possible_map(void); -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ /* Sparc specific messages. */ #define MSG_CROSS_CALL 0x0005 /* run func on cpus */ diff --git a/arch/sparc/include/asm/smp_64.h b/arch/sparc/include/asm/smp_64.h index dd3bef4b9896..759fb4a9530e 100644 --- a/arch/sparc/include/asm/smp_64.h +++ b/arch/sparc/include/asm/smp_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* smp.h: Sparc64 specific SMP stuff. * * Copyright (C) 1996, 2008 David S. Miller (davem@davemloft.net) @@ -11,16 +12,16 @@ #include <asm/starfire.h> #include <asm/spitfire.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/cpumask.h> #include <linux/cache.h> -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #ifdef CONFIG_SMP -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * Private routines/data @@ -32,34 +33,42 @@ DECLARE_PER_CPU(cpumask_t, cpu_sibling_map); extern cpumask_t cpu_core_map[NR_CPUS]; -extern int sparc64_multi_core; -extern void arch_send_call_function_single_ipi(int cpu); -extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); +void smp_init_cpu_poke(void); +void scheduler_poke(void); + +void arch_send_call_function_single_ipi(int cpu); +void arch_send_call_function_ipi_mask(const struct cpumask *mask); /* * General functions that each host system must provide. */ -extern int hard_smp_processor_id(void); +int hard_smp_processor_id(void); #define raw_smp_processor_id() (current_thread_info()->cpu) -extern void smp_fill_in_sib_core_maps(void); -extern void cpu_play_dead(void); +void smp_fill_in_sib_core_maps(void); +void __noreturn cpu_play_dead(void); -extern void smp_fetch_global_regs(void); -extern void smp_fetch_global_pmu(void); +void smp_fetch_global_regs(void); +void smp_fetch_global_pmu(void); struct seq_file; void smp_bogo(struct seq_file *); void smp_info(struct seq_file *); +void smp_callin(void); +void cpu_panic(void); +void smp_synchronize_tick_client(void); +void smp_capture(void); +void smp_release(void); + #ifdef CONFIG_HOTPLUG_CPU -extern int __cpu_disable(void); -extern void __cpu_die(unsigned int cpu); +int __cpu_disable(void); +void __cpu_die(unsigned int cpu); #endif -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #else @@ -67,6 +76,8 @@ extern void __cpu_die(unsigned int cpu); #define smp_fill_in_sib_core_maps() do { } while (0) #define smp_fetch_global_regs() do { } while (0) #define smp_fetch_global_pmu() do { } while (0) +#define smp_init_cpu_poke() do { } while (0) +#define scheduler_poke() do { } while (0) #endif /* !(CONFIG_SMP) */ diff --git a/arch/sparc/include/asm/sparsemem.h b/arch/sparc/include/asm/sparsemem.h index b99d4e4b6d28..aa9a676bc341 100644 --- a/arch/sparc/include/asm/sparsemem.h +++ b/arch/sparc/include/asm/sparsemem.h @@ -1,11 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_SPARSEMEM_H #define _SPARC64_SPARSEMEM_H #ifdef __KERNEL__ +#include <asm/page.h> + #define SECTION_SIZE_BITS 30 -#define MAX_PHYSADDR_BITS 42 -#define MAX_PHYSMEM_BITS 42 +#define MAX_PHYSMEM_BITS MAX_PHYS_ADDRESS_BITS #endif /* !(__KERNEL__) */ diff --git a/arch/sparc/include/asm/spinlock.h b/arch/sparc/include/asm/spinlock.h index f276b0036b2c..3f4ce55bc4d6 100644 --- a/arch/sparc/include/asm/spinlock.h +++ b/arch/sparc/include/asm/spinlock.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_SPINLOCK_H #define ___ASM_SPARC_SPINLOCK_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/spinlock_32.h b/arch/sparc/include/asm/spinlock_32.h index bcc98fc35281..6d6d261bf8d2 100644 --- a/arch/sparc/include/asm/spinlock_32.h +++ b/arch/sparc/include/asm/spinlock_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* spinlock.h: 32-bit Sparc spinlock support. * * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) @@ -6,16 +7,14 @@ #ifndef __SPARC_SPINLOCK_H #define __SPARC_SPINLOCK_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/psr.h> +#include <asm/barrier.h> #include <asm/processor.h> /* for cpu_relax */ #define arch_spin_is_locked(lock) (*((volatile unsigned char *)(lock)) != 0) -#define arch_spin_unlock_wait(lock) \ - do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) - static inline void arch_spin_lock(arch_spinlock_t *lock) { __asm__ __volatile__( @@ -131,7 +130,7 @@ static inline void arch_write_lock(arch_rwlock_t *rw) *(volatile __u32 *)&lp->lock = ~0U; } -static void inline arch_write_unlock(arch_rwlock_t *lock) +static inline void arch_write_unlock(arch_rwlock_t *lock) { __asm__ __volatile__( " st %%g0, [%0]" @@ -184,17 +183,6 @@ static inline int __arch_read_trylock(arch_rwlock_t *rw) res; \ }) -#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) -#define arch_read_lock_flags(rw, flags) arch_read_lock(rw) -#define arch_write_lock_flags(rw, flags) arch_write_lock(rw) - -#define arch_spin_relax(lock) cpu_relax() -#define arch_read_relax(lock) cpu_relax() -#define arch_write_relax(lock) cpu_relax() - -#define arch_read_can_lock(rw) (!((rw)->lock & 0xff)) -#define arch_write_can_lock(rw) (!(rw)->lock) - -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #endif /* __SPARC_SPINLOCK_H */ diff --git a/arch/sparc/include/asm/spinlock_64.h b/arch/sparc/include/asm/spinlock_64.h index 968917694978..13cd15d346be 100644 --- a/arch/sparc/include/asm/spinlock_64.h +++ b/arch/sparc/include/asm/spinlock_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* spinlock.h: 64-bit Sparc spinlock support. * * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) @@ -6,220 +7,13 @@ #ifndef __SPARC64_SPINLOCK_H #define __SPARC64_SPINLOCK_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ -/* To get debugging spinlocks which detect and catch - * deadlock situations, set CONFIG_DEBUG_SPINLOCK - * and rebuild your kernel. - */ - -/* Because we play games to save cycles in the non-contention case, we - * need to be extra careful about branch targets into the "spinning" - * code. They live in their own section, but the newer V9 branches - * have a shorter range than the traditional 32-bit sparc branch - * variants. The rule is that the branches that go into and out of - * the spinner sections must be pre-V9 branches. - */ - -#define arch_spin_is_locked(lp) ((lp)->lock != 0) - -#define arch_spin_unlock_wait(lp) \ - do { rmb(); \ - } while((lp)->lock) - -static inline void arch_spin_lock(arch_spinlock_t *lock) -{ - unsigned long tmp; - - __asm__ __volatile__( -"1: ldstub [%1], %0\n" -" brnz,pn %0, 2f\n" -" nop\n" -" .subsection 2\n" -"2: ldub [%1], %0\n" -" brnz,pt %0, 2b\n" -" nop\n" -" ba,a,pt %%xcc, 1b\n" -" .previous" - : "=&r" (tmp) - : "r" (lock) - : "memory"); -} - -static inline int arch_spin_trylock(arch_spinlock_t *lock) -{ - unsigned long result; - - __asm__ __volatile__( -" ldstub [%1], %0\n" - : "=r" (result) - : "r" (lock) - : "memory"); - - return (result == 0UL); -} - -static inline void arch_spin_unlock(arch_spinlock_t *lock) -{ - __asm__ __volatile__( -" stb %%g0, [%0]" - : /* No outputs */ - : "r" (lock) - : "memory"); -} - -static inline void arch_spin_lock_flags(arch_spinlock_t *lock, unsigned long flags) -{ - unsigned long tmp1, tmp2; - - __asm__ __volatile__( -"1: ldstub [%2], %0\n" -" brnz,pn %0, 2f\n" -" nop\n" -" .subsection 2\n" -"2: rdpr %%pil, %1\n" -" wrpr %3, %%pil\n" -"3: ldub [%2], %0\n" -" brnz,pt %0, 3b\n" -" nop\n" -" ba,pt %%xcc, 1b\n" -" wrpr %1, %%pil\n" -" .previous" - : "=&r" (tmp1), "=&r" (tmp2) - : "r"(lock), "r"(flags) - : "memory"); -} - -/* Multi-reader locks, these are much saner than the 32-bit Sparc ones... */ - -static void inline arch_read_lock(arch_rwlock_t *lock) -{ - unsigned long tmp1, tmp2; - - __asm__ __volatile__ ( -"1: ldsw [%2], %0\n" -" brlz,pn %0, 2f\n" -"4: add %0, 1, %1\n" -" cas [%2], %0, %1\n" -" cmp %0, %1\n" -" bne,pn %%icc, 1b\n" -" nop\n" -" .subsection 2\n" -"2: ldsw [%2], %0\n" -" brlz,pt %0, 2b\n" -" nop\n" -" ba,a,pt %%xcc, 4b\n" -" .previous" - : "=&r" (tmp1), "=&r" (tmp2) - : "r" (lock) - : "memory"); -} - -static int inline arch_read_trylock(arch_rwlock_t *lock) -{ - int tmp1, tmp2; - - __asm__ __volatile__ ( -"1: ldsw [%2], %0\n" -" brlz,a,pn %0, 2f\n" -" mov 0, %0\n" -" add %0, 1, %1\n" -" cas [%2], %0, %1\n" -" cmp %0, %1\n" -" bne,pn %%icc, 1b\n" -" mov 1, %0\n" -"2:" - : "=&r" (tmp1), "=&r" (tmp2) - : "r" (lock) - : "memory"); - - return tmp1; -} - -static void inline arch_read_unlock(arch_rwlock_t *lock) -{ - unsigned long tmp1, tmp2; - - __asm__ __volatile__( -"1: lduw [%2], %0\n" -" sub %0, 1, %1\n" -" cas [%2], %0, %1\n" -" cmp %0, %1\n" -" bne,pn %%xcc, 1b\n" -" nop" - : "=&r" (tmp1), "=&r" (tmp2) - : "r" (lock) - : "memory"); -} - -static void inline arch_write_lock(arch_rwlock_t *lock) -{ - unsigned long mask, tmp1, tmp2; - - mask = 0x80000000UL; - - __asm__ __volatile__( -"1: lduw [%2], %0\n" -" brnz,pn %0, 2f\n" -"4: or %0, %3, %1\n" -" cas [%2], %0, %1\n" -" cmp %0, %1\n" -" bne,pn %%icc, 1b\n" -" nop\n" -" .subsection 2\n" -"2: lduw [%2], %0\n" -" brnz,pt %0, 2b\n" -" nop\n" -" ba,a,pt %%xcc, 4b\n" -" .previous" - : "=&r" (tmp1), "=&r" (tmp2) - : "r" (lock), "r" (mask) - : "memory"); -} - -static void inline arch_write_unlock(arch_rwlock_t *lock) -{ - __asm__ __volatile__( -" stw %%g0, [%0]" - : /* no outputs */ - : "r" (lock) - : "memory"); -} - -static int inline arch_write_trylock(arch_rwlock_t *lock) -{ - unsigned long mask, tmp1, tmp2, result; - - mask = 0x80000000UL; - - __asm__ __volatile__( -" mov 0, %2\n" -"1: lduw [%3], %0\n" -" brnz,pn %0, 2f\n" -" or %0, %4, %1\n" -" cas [%3], %0, %1\n" -" cmp %0, %1\n" -" bne,pn %%icc, 1b\n" -" nop\n" -" mov 1, %2\n" -"2:" - : "=&r" (tmp1), "=&r" (tmp2), "=&r" (result) - : "r" (lock), "r" (mask) - : "memory"); - - return result; -} - -#define arch_read_lock_flags(p, f) arch_read_lock(p) -#define arch_write_lock_flags(p, f) arch_write_lock(p) - -#define arch_read_can_lock(rw) (!((rw)->lock & 0x80000000UL)) -#define arch_write_can_lock(rw) (!(rw)->lock) - -#define arch_spin_relax(lock) cpu_relax() -#define arch_read_relax(lock) cpu_relax() -#define arch_write_relax(lock) cpu_relax() +#include <asm/processor.h> +#include <asm/barrier.h> +#include <asm/qspinlock.h> +#include <asm/qrwlock.h> -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #endif /* !(__SPARC64_SPINLOCK_H) */ diff --git a/arch/sparc/include/asm/spinlock_types.h b/arch/sparc/include/asm/spinlock_types.h index 9c454fdeaad8..ed1d569b638e 100644 --- a/arch/sparc/include/asm/spinlock_types.h +++ b/arch/sparc/include/asm/spinlock_types.h @@ -1,20 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_SPINLOCK_TYPES_H #define __SPARC_SPINLOCK_TYPES_H -#ifndef __LINUX_SPINLOCK_TYPES_H -# error "please don't include this file directly" -#endif +#ifdef CONFIG_QUEUED_SPINLOCKS +#include <asm-generic/qspinlock_types.h> +#else typedef struct { volatile unsigned char lock; } arch_spinlock_t; #define __ARCH_SPIN_LOCK_UNLOCKED { 0 } +#endif /* CONFIG_QUEUED_SPINLOCKS */ +#ifdef CONFIG_QUEUED_RWLOCKS +#include <asm-generic/qrwlock_types.h> +#else typedef struct { volatile unsigned int lock; } arch_rwlock_t; #define __ARCH_RW_LOCK_UNLOCKED { 0 } - +#endif /* CONFIG_QUEUED_RWLOCKS */ #endif diff --git a/arch/sparc/include/asm/spitfire.h b/arch/sparc/include/asm/spitfire.h index 6b67e50fb9b4..79b9dd5e9ac6 100644 --- a/arch/sparc/include/asm/spitfire.h +++ b/arch/sparc/include/asm/spitfire.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* spitfire.h: SpitFire/BlackBird/Cheetah inline MMU operations. * * Copyright (C) 1996 David S. Miller (davem@davemloft.net) @@ -45,10 +46,29 @@ #define SUN4V_CHIP_NIAGARA3 0x03 #define SUN4V_CHIP_NIAGARA4 0x04 #define SUN4V_CHIP_NIAGARA5 0x05 +#define SUN4V_CHIP_SPARC_M6 0x06 +#define SUN4V_CHIP_SPARC_M7 0x07 +#define SUN4V_CHIP_SPARC_M8 0x08 #define SUN4V_CHIP_SPARC64X 0x8a +#define SUN4V_CHIP_SPARC_SN 0x8b #define SUN4V_CHIP_UNKNOWN 0xff -#ifndef __ASSEMBLY__ +/* + * The following CPU_ID_xxx constants are used + * to identify the CPU type in the setup phase + * (see head_64.S) + */ +#define CPU_ID_NIAGARA1 ('1') +#define CPU_ID_NIAGARA2 ('2') +#define CPU_ID_NIAGARA3 ('3') +#define CPU_ID_NIAGARA4 ('4') +#define CPU_ID_NIAGARA5 ('5') +#define CPU_ID_M6 ('6') +#define CPU_ID_M7 ('7') +#define CPU_ID_M8 ('8') +#define CPU_ID_SONOMA1 ('N') + +#ifndef __ASSEMBLER__ enum ultra_tlb_layout { spitfire = 0, @@ -62,7 +82,7 @@ extern enum ultra_tlb_layout tlb_type; extern int sun4v_chip_type; extern int cheetah_pcache_forced_on; -extern void cheetah_enable_pcache(void); +void cheetah_enable_pcache(void); #define sparc64_highest_locked_tlbent() \ (tlb_type == spitfire ? \ @@ -343,6 +363,6 @@ static inline void cheetah_put_itlb_data(int entry, unsigned long data) "i" (ASI_ITLB_DATA_ACCESS)); } -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #endif /* CONFIG_SPARC64 */ #endif /* !(_SPARC64_SPITFIRE_H) */ diff --git a/arch/sparc/include/asm/stacktrace.h b/arch/sparc/include/asm/stacktrace.h index 6cee39adf6d6..556ec5d59c88 100644 --- a/arch/sparc/include/asm/stacktrace.h +++ b/arch/sparc/include/asm/stacktrace.h @@ -1,6 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_STACKTRACE_H #define _SPARC64_STACKTRACE_H -extern void stack_trace_flush(void); +void stack_trace_flush(void); #endif /* _SPARC64_STACKTRACE_H */ diff --git a/arch/sparc/include/asm/starfire.h b/arch/sparc/include/asm/starfire.h index d56ce60a5992..8e511ed78775 100644 --- a/arch/sparc/include/asm/starfire.h +++ b/arch/sparc/include/asm/starfire.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * starfire.h: Group all starfire specific code together. * @@ -7,14 +8,13 @@ #ifndef _SPARC64_STARFIRE_H #define _SPARC64_STARFIRE_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ extern int this_is_starfire; -extern void check_if_starfire(void); -extern int starfire_hard_smp_processor_id(void); -extern void starfire_hookup(int); -extern unsigned int starfire_translate(unsigned long imap, unsigned int upaid); +void check_if_starfire(void); +void starfire_hookup(int); +unsigned int starfire_translate(unsigned long imap, unsigned int upaid); #endif #endif diff --git a/arch/sparc/include/asm/string.h b/arch/sparc/include/asm/string.h index 98b72a0c8e6e..001a17baf2d5 100644 --- a/arch/sparc/include/asm/string.h +++ b/arch/sparc/include/asm/string.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_STRING_H #define ___ASM_SPARC_STRING_H #if defined(__sparc__) && defined(__arch64__) @@ -5,4 +6,42 @@ #else #include <asm/string_32.h> #endif + +/* First the mem*() things. */ +#define __HAVE_ARCH_MEMMOVE +void *memmove(void *, const void *, __kernel_size_t); + +#define __HAVE_ARCH_MEMCPY +#define memcpy(t, f, n) __builtin_memcpy(t, f, n) + +#define __HAVE_ARCH_MEMSET +#define memset(s, c, count) __builtin_memset(s, c, count) + +#define __HAVE_ARCH_MEMSCAN + +#define memscan(__arg0, __char, __arg2) \ +({ \ + void *__memscan_zero(void *, size_t); \ + void *__memscan_generic(void *, int, size_t); \ + void *__retval, *__addr = (__arg0); \ + size_t __size = (__arg2); \ + \ + if(__builtin_constant_p(__char) && !(__char)) \ + __retval = __memscan_zero(__addr, __size); \ + else \ + __retval = __memscan_generic(__addr, (__char), __size); \ + \ + __retval; \ +}) + +#define __HAVE_ARCH_MEMCMP +int memcmp(const void *,const void *,__kernel_size_t); + +/* Now the str*() stuff... */ +#define __HAVE_ARCH_STRLEN +__kernel_size_t strlen(const char *); + +#define __HAVE_ARCH_STRNCMP +int strncmp(const char *, const char *, __kernel_size_t); + #endif diff --git a/arch/sparc/include/asm/string_32.h b/arch/sparc/include/asm/string_32.h index 12f67857152e..f488946bd7d6 100644 --- a/arch/sparc/include/asm/string_32.h +++ b/arch/sparc/include/asm/string_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * string.h: External definitions for optimized assembly string * routines for the Linux Kernel. @@ -11,60 +12,4 @@ #include <asm/page.h> -/* Really, userland/ksyms should not see any of this stuff. */ - -#ifdef __KERNEL__ - -extern void __memmove(void *,const void *,__kernel_size_t); - -#ifndef EXPORT_SYMTAB_STROPS - -/* First the mem*() things. */ -#define __HAVE_ARCH_MEMMOVE -#undef memmove -#define memmove(_to, _from, _n) \ -({ \ - void *_t = (_to); \ - __memmove(_t, (_from), (_n)); \ - _t; \ -}) - -#define __HAVE_ARCH_MEMCPY -#define memcpy(t, f, n) __builtin_memcpy(t, f, n) - -#define __HAVE_ARCH_MEMSET -#define memset(s, c, count) __builtin_memset(s, c, count) - -#define __HAVE_ARCH_MEMSCAN - -#undef memscan -#define memscan(__arg0, __char, __arg2) \ -({ \ - extern void *__memscan_zero(void *, size_t); \ - extern void *__memscan_generic(void *, int, size_t); \ - void *__retval, *__addr = (__arg0); \ - size_t __size = (__arg2); \ - \ - if(__builtin_constant_p(__char) && !(__char)) \ - __retval = __memscan_zero(__addr, __size); \ - else \ - __retval = __memscan_generic(__addr, (__char), __size); \ - \ - __retval; \ -}) - -#define __HAVE_ARCH_MEMCMP -extern int memcmp(const void *,const void *,__kernel_size_t); - -/* Now the str*() stuff... */ -#define __HAVE_ARCH_STRLEN -extern __kernel_size_t strlen(const char *); - -#define __HAVE_ARCH_STRNCMP -extern int strncmp(const char *, const char *, __kernel_size_t); - -#endif /* !EXPORT_SYMTAB_STROPS */ - -#endif /* __KERNEL__ */ - #endif /* !(__SPARC_STRING_H__) */ diff --git a/arch/sparc/include/asm/string_64.h b/arch/sparc/include/asm/string_64.h index 9623bc213158..d5c563058a5b 100644 --- a/arch/sparc/include/asm/string_64.h +++ b/arch/sparc/include/asm/string_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * string.h: External definitions for optimized assembly string * routines for the Linux Kernel. @@ -9,54 +10,6 @@ #ifndef __SPARC64_STRING_H__ #define __SPARC64_STRING_H__ -/* Really, userland/ksyms should not see any of this stuff. */ - -#ifdef __KERNEL__ - #include <asm/asi.h> -#ifndef EXPORT_SYMTAB_STROPS - -/* First the mem*() things. */ -#define __HAVE_ARCH_MEMMOVE -extern void *memmove(void *, const void *, __kernel_size_t); - -#define __HAVE_ARCH_MEMCPY -#define memcpy(t, f, n) __builtin_memcpy(t, f, n) - -#define __HAVE_ARCH_MEMSET -#define memset(s, c, count) __builtin_memset(s, c, count) - -#define __HAVE_ARCH_MEMSCAN - -#undef memscan -#define memscan(__arg0, __char, __arg2) \ -({ \ - extern void *__memscan_zero(void *, size_t); \ - extern void *__memscan_generic(void *, int, size_t); \ - void *__retval, *__addr = (__arg0); \ - size_t __size = (__arg2); \ - \ - if(__builtin_constant_p(__char) && !(__char)) \ - __retval = __memscan_zero(__addr, __size); \ - else \ - __retval = __memscan_generic(__addr, (__char), __size); \ - \ - __retval; \ -}) - -#define __HAVE_ARCH_MEMCMP -extern int memcmp(const void *,const void *,__kernel_size_t); - -/* Now the str*() stuff... */ -#define __HAVE_ARCH_STRLEN -extern __kernel_size_t strlen(const char *); - -#define __HAVE_ARCH_STRNCMP -extern int strncmp(const char *, const char *, __kernel_size_t); - -#endif /* !EXPORT_SYMTAB_STROPS */ - -#endif /* __KERNEL__ */ - #endif /* !(__SPARC64_STRING_H__) */ diff --git a/arch/sparc/include/asm/sunbpp.h b/arch/sparc/include/asm/sunbpp.h index d81a02eaf78b..55de4da70dff 100644 --- a/arch/sparc/include/asm/sunbpp.h +++ b/arch/sparc/include/asm/sunbpp.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * include/asm/sunbpp.h */ diff --git a/arch/sparc/include/asm/swift.h b/arch/sparc/include/asm/swift.h index e535061bf755..96f6526b964e 100644 --- a/arch/sparc/include/asm/swift.h +++ b/arch/sparc/include/asm/swift.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* swift.h: Specific definitions for the _broken_ Swift SRMMU * MMU module. * diff --git a/arch/sparc/include/asm/switch_to.h b/arch/sparc/include/asm/switch_to.h index 2dc4fa5c6f8c..7cf1c5dc734c 100644 --- a/arch/sparc/include/asm/switch_to.h +++ b/arch/sparc/include/asm/switch_to.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_SWITCH_TO_H #define ___ASM_SPARC_SWITCH_TO_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/switch_to_32.h b/arch/sparc/include/asm/switch_to_32.h index e32e82b76eed..42eeafcb8a41 100644 --- a/arch/sparc/include/asm/switch_to_32.h +++ b/arch/sparc/include/asm/switch_to_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_SWITCH_TO_H #define __SPARC_SWITCH_TO_H @@ -9,7 +10,7 @@ extern struct thread_info *current_set[NR_CPUS]; * Flush windows so that the VM switch which follows * would not pull the stack from under us. * - * SWITCH_ENTER and SWITH_DO_LAZY_FPU do not work yet (e.g. SMP does not work) + * SWITCH_ENTER and SWITCH_DO_LAZY_FPU do not work yet (e.g. SMP does not work) * XXX WTF is the above comment? Found in late teen 2.4.x. */ #ifdef CONFIG_SMP @@ -99,8 +100,8 @@ extern struct thread_info *current_set[NR_CPUS]; "o0", "o1", "o2", "o3", "o7"); \ } while(0) -extern void fpsave(unsigned long *fpregs, unsigned long *fsr, - void *fpqueue, unsigned long *fpqdepth); -extern void synchronize_user_stack(void); +void fpsave(unsigned long *fpregs, unsigned long *fsr, + void *fpqueue, unsigned long *fpqdepth); +void synchronize_user_stack(void); #endif /* __SPARC_SWITCH_TO_H */ diff --git a/arch/sparc/include/asm/switch_to_64.h b/arch/sparc/include/asm/switch_to_64.h index c7de3323819c..d93963ff7caa 100644 --- a/arch/sparc/include/asm/switch_to_64.h +++ b/arch/sparc/include/asm/switch_to_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC64_SWITCH_TO_64_H #define __SPARC64_SWITCH_TO_64_H @@ -14,15 +15,13 @@ do { \ * for l0/l1. It will use one for 'next' and the other to hold * the output value of 'last'. 'next' is not referenced again * past the invocation of switch_to in the scheduler, so we need - * not preserve it's value. Hairy, but it lets us remove 2 loads + * not preserve its value. Hairy, but it lets us remove 2 loads * and 2 stores in this critical code path. -DaveM */ #define switch_to(prev, next, last) \ do { save_and_clear_fpu(); \ - /* If you are tempted to conditionalize the following */ \ - /* so that ASI is only written if it changes, think again. */ \ __asm__ __volatile__("wr %%g0, %0, %%asi" \ - : : "r" (task_thread_info(next)->current_ds));\ + : : "r" (ASI_AIUS)); \ trap_block[current_thread_info()->cpu].thread = \ task_thread_info(next); \ __asm__ __volatile__( \ @@ -48,8 +47,8 @@ do { save_and_clear_fpu(); \ "wrpr %%g0, 14, %%pil\n\t" \ "brz,pt %%o7, switch_to_pc\n\t" \ " mov %%g7, %0\n\t" \ - "sethi %%hi(ret_from_syscall), %%g1\n\t" \ - "jmpl %%g1 + %%lo(ret_from_syscall), %%g0\n\t" \ + "sethi %%hi(ret_from_fork), %%g1\n\t" \ + "jmpl %%g1 + %%lo(ret_from_fork), %%g0\n\t" \ " nop\n\t" \ ".globl switch_to_pc\n\t" \ "switch_to_pc:\n\t" \ @@ -65,7 +64,8 @@ do { save_and_clear_fpu(); \ "o0", "o1", "o2", "o3", "o4", "o5", "o7"); \ } while(0) -extern void synchronize_user_stack(void); -extern void fault_in_user_windows(void); +void synchronize_user_stack(void); +struct pt_regs; +void fault_in_user_windows(struct pt_regs *); #endif /* __SPARC64_SWITCH_TO_64_H */ diff --git a/arch/sparc/include/asm/syscall.h b/arch/sparc/include/asm/syscall.h index 025a02ad2e31..b0233924d323 100644 --- a/arch/sparc/include/asm/syscall.h +++ b/arch/sparc/include/asm/syscall.h @@ -1,9 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASM_SPARC_SYSCALL_H #define __ASM_SPARC_SYSCALL_H +#include <uapi/linux/audit.h> #include <linux/kernel.h> +#include <linux/compat.h> #include <linux/sched.h> #include <asm/ptrace.h> +#include <asm/thread_info.h> /* * The syscall table always contains 32 bit pointers since we know that the @@ -21,6 +25,18 @@ static inline long syscall_get_nr(struct task_struct *task, return (syscall_p ? regs->u_regs[UREG_G1] : -1L); } +static inline void syscall_set_nr(struct task_struct *task, + struct pt_regs *regs, + int nr) +{ + /* + * Unlike syscall_get_nr(), syscall_set_nr() can be called only when + * the target task is stopped for tracing on entering syscall, so + * there is no need to have the same check syscall_get_nr() has. + */ + regs->u_regs[UREG_G1] = nr; +} + static inline void syscall_rollback(struct task_struct *task, struct pt_regs *regs) { @@ -92,11 +108,11 @@ static inline void syscall_set_return_value(struct task_struct *task, static inline void syscall_get_arguments(struct task_struct *task, struct pt_regs *regs, - unsigned int i, unsigned int n, unsigned long *args) { int zero_extend = 0; unsigned int j; + unsigned int n = 6; #ifdef CONFIG_SPARC64 if (test_tsk_thread_flag(task, TIF_32BIT)) @@ -104,7 +120,7 @@ static inline void syscall_get_arguments(struct task_struct *task, #endif for (j = 0; j < n; j++) { - unsigned long val = regs->u_regs[UREG_I0 + i + j]; + unsigned long val = regs->u_regs[UREG_I0 + j]; if (zero_extend) args[j] = (u32) val; @@ -115,13 +131,24 @@ static inline void syscall_get_arguments(struct task_struct *task, static inline void syscall_set_arguments(struct task_struct *task, struct pt_regs *regs, - unsigned int i, unsigned int n, const unsigned long *args) { - unsigned int j; + unsigned int i; - for (j = 0; j < n; j++) - regs->u_regs[UREG_I0 + i + j] = args[j]; + for (i = 0; i < 6; i++) + regs->u_regs[UREG_I0 + i] = args[i]; +} + +static inline int syscall_get_arch(struct task_struct *task) +{ +#if defined(CONFIG_SPARC64) && defined(CONFIG_COMPAT) + return test_tsk_thread_flag(task, TIF_32BIT) + ? AUDIT_ARCH_SPARC : AUDIT_ARCH_SPARC64; +#elif defined(CONFIG_SPARC64) + return AUDIT_ARCH_SPARC64; +#else + return AUDIT_ARCH_SPARC; +#endif } #endif /* __ASM_SPARC_SYSCALL_H */ diff --git a/arch/sparc/include/asm/syscalls.h b/arch/sparc/include/asm/syscalls.h index bf8972adea17..35575fbfb9dc 100644 --- a/arch/sparc/include/asm/syscalls.h +++ b/arch/sparc/include/asm/syscalls.h @@ -1,11 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_SYSCALLS_H #define _SPARC64_SYSCALLS_H struct pt_regs; -extern asmlinkage long sparc_do_fork(unsigned long clone_flags, - unsigned long stack_start, - struct pt_regs *regs, - unsigned long stack_size); +asmlinkage long sparc_fork(struct pt_regs *regs); +asmlinkage long sparc_vfork(struct pt_regs *regs); +asmlinkage long sparc_clone(struct pt_regs *regs); #endif /* _SPARC64_SYSCALLS_H */ diff --git a/arch/sparc/include/asm/termbits.h b/arch/sparc/include/asm/termbits.h index 948067065ac5..fa9de4a46d36 100644 --- a/arch/sparc/include/asm/termbits.h +++ b/arch/sparc/include/asm/termbits.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_TERMBITS_H #define _SPARC_TERMBITS_H diff --git a/arch/sparc/include/asm/termios.h b/arch/sparc/include/asm/termios.h deleted file mode 100644 index 0c2414ddd52c..000000000000 --- a/arch/sparc/include/asm/termios.h +++ /dev/null @@ -1,146 +0,0 @@ -#ifndef _SPARC_TERMIOS_H -#define _SPARC_TERMIOS_H - -#include <uapi/asm/termios.h> - - -/* - * c_cc characters in the termio structure. Oh, how I love being - * backwardly compatible. Notice that character 4 and 5 are - * interpreted differently depending on whether ICANON is set in - * c_lflag. If it's set, they are used as _VEOF and _VEOL, otherwise - * as _VMIN and V_TIME. This is for compatibility with OSF/1 (which - * is compatible with sysV)... - */ -#define _VMIN 4 -#define _VTIME 5 - -/* intr=^C quit=^\ erase=del kill=^U - eof=^D eol=\0 eol2=\0 sxtc=\0 - start=^Q stop=^S susp=^Z dsusp=^Y - reprint=^R discard=^U werase=^W lnext=^V - vmin=\1 vtime=\0 -*/ -#define INIT_C_CC "\003\034\177\025\004\000\000\000\021\023\032\031\022\025\027\026\001" - -/* - * Translate a "termio" structure into a "termios". Ugh. - */ -#define user_termio_to_kernel_termios(termios, termio) \ -({ \ - unsigned short tmp; \ - int err; \ - err = get_user(tmp, &(termio)->c_iflag); \ - (termios)->c_iflag = (0xffff0000 & ((termios)->c_iflag)) | tmp; \ - err |= get_user(tmp, &(termio)->c_oflag); \ - (termios)->c_oflag = (0xffff0000 & ((termios)->c_oflag)) | tmp; \ - err |= get_user(tmp, &(termio)->c_cflag); \ - (termios)->c_cflag = (0xffff0000 & ((termios)->c_cflag)) | tmp; \ - err |= get_user(tmp, &(termio)->c_lflag); \ - (termios)->c_lflag = (0xffff0000 & ((termios)->c_lflag)) | tmp; \ - err |= copy_from_user((termios)->c_cc, (termio)->c_cc, NCC); \ - err; \ -}) - -/* - * Translate a "termios" structure into a "termio". Ugh. - * - * Note the "fun" _VMIN overloading. - */ -#define kernel_termios_to_user_termio(termio, termios) \ -({ \ - int err; \ - err = put_user((termios)->c_iflag, &(termio)->c_iflag); \ - err |= put_user((termios)->c_oflag, &(termio)->c_oflag); \ - err |= put_user((termios)->c_cflag, &(termio)->c_cflag); \ - err |= put_user((termios)->c_lflag, &(termio)->c_lflag); \ - err |= put_user((termios)->c_line, &(termio)->c_line); \ - err |= copy_to_user((termio)->c_cc, (termios)->c_cc, NCC); \ - if (!((termios)->c_lflag & ICANON)) { \ - err |= put_user((termios)->c_cc[VMIN], &(termio)->c_cc[_VMIN]); \ - err |= put_user((termios)->c_cc[VTIME], &(termio)->c_cc[_VTIME]); \ - } \ - err; \ -}) - -#define user_termios_to_kernel_termios(k, u) \ -({ \ - int err; \ - err = get_user((k)->c_iflag, &(u)->c_iflag); \ - err |= get_user((k)->c_oflag, &(u)->c_oflag); \ - err |= get_user((k)->c_cflag, &(u)->c_cflag); \ - err |= get_user((k)->c_lflag, &(u)->c_lflag); \ - err |= get_user((k)->c_line, &(u)->c_line); \ - err |= copy_from_user((k)->c_cc, (u)->c_cc, NCCS); \ - if ((k)->c_lflag & ICANON) { \ - err |= get_user((k)->c_cc[VEOF], &(u)->c_cc[VEOF]); \ - err |= get_user((k)->c_cc[VEOL], &(u)->c_cc[VEOL]); \ - } else { \ - err |= get_user((k)->c_cc[VMIN], &(u)->c_cc[_VMIN]); \ - err |= get_user((k)->c_cc[VTIME], &(u)->c_cc[_VTIME]); \ - } \ - err |= get_user((k)->c_ispeed, &(u)->c_ispeed); \ - err |= get_user((k)->c_ospeed, &(u)->c_ospeed); \ - err; \ -}) - -#define kernel_termios_to_user_termios(u, k) \ -({ \ - int err; \ - err = put_user((k)->c_iflag, &(u)->c_iflag); \ - err |= put_user((k)->c_oflag, &(u)->c_oflag); \ - err |= put_user((k)->c_cflag, &(u)->c_cflag); \ - err |= put_user((k)->c_lflag, &(u)->c_lflag); \ - err |= put_user((k)->c_line, &(u)->c_line); \ - err |= copy_to_user((u)->c_cc, (k)->c_cc, NCCS); \ - if (!((k)->c_lflag & ICANON)) { \ - err |= put_user((k)->c_cc[VMIN], &(u)->c_cc[_VMIN]); \ - err |= put_user((k)->c_cc[VTIME], &(u)->c_cc[_VTIME]); \ - } else { \ - err |= put_user((k)->c_cc[VEOF], &(u)->c_cc[VEOF]); \ - err |= put_user((k)->c_cc[VEOL], &(u)->c_cc[VEOL]); \ - } \ - err |= put_user((k)->c_ispeed, &(u)->c_ispeed); \ - err |= put_user((k)->c_ospeed, &(u)->c_ospeed); \ - err; \ -}) - -#define user_termios_to_kernel_termios_1(k, u) \ -({ \ - int err; \ - err = get_user((k)->c_iflag, &(u)->c_iflag); \ - err |= get_user((k)->c_oflag, &(u)->c_oflag); \ - err |= get_user((k)->c_cflag, &(u)->c_cflag); \ - err |= get_user((k)->c_lflag, &(u)->c_lflag); \ - err |= get_user((k)->c_line, &(u)->c_line); \ - err |= copy_from_user((k)->c_cc, (u)->c_cc, NCCS); \ - if ((k)->c_lflag & ICANON) { \ - err |= get_user((k)->c_cc[VEOF], &(u)->c_cc[VEOF]); \ - err |= get_user((k)->c_cc[VEOL], &(u)->c_cc[VEOL]); \ - } else { \ - err |= get_user((k)->c_cc[VMIN], &(u)->c_cc[_VMIN]); \ - err |= get_user((k)->c_cc[VTIME], &(u)->c_cc[_VTIME]); \ - } \ - err; \ -}) - -#define kernel_termios_to_user_termios_1(u, k) \ -({ \ - int err; \ - err = put_user((k)->c_iflag, &(u)->c_iflag); \ - err |= put_user((k)->c_oflag, &(u)->c_oflag); \ - err |= put_user((k)->c_cflag, &(u)->c_cflag); \ - err |= put_user((k)->c_lflag, &(u)->c_lflag); \ - err |= put_user((k)->c_line, &(u)->c_line); \ - err |= copy_to_user((u)->c_cc, (k)->c_cc, NCCS); \ - if (!((k)->c_lflag & ICANON)) { \ - err |= put_user((k)->c_cc[VMIN], &(u)->c_cc[_VMIN]); \ - err |= put_user((k)->c_cc[VTIME], &(u)->c_cc[_VTIME]); \ - } else { \ - err |= put_user((k)->c_cc[VEOF], &(u)->c_cc[VEOF]); \ - err |= put_user((k)->c_cc[VEOL], &(u)->c_cc[VEOL]); \ - } \ - err; \ -}) - -#endif /* _SPARC_TERMIOS_H */ diff --git a/arch/sparc/include/asm/thread_info.h b/arch/sparc/include/asm/thread_info.h index 122d7acc07e6..740b1743ed8c 100644 --- a/arch/sparc/include/asm/thread_info.h +++ b/arch/sparc/include/asm/thread_info.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_THREAD_INFO_H #define ___ASM_SPARC_THREAD_INFO_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/thread_info_32.h b/arch/sparc/include/asm/thread_info_32.h index dd3807599bb9..fdaf7b171e0a 100644 --- a/arch/sparc/include/asm/thread_info_32.h +++ b/arch/sparc/include/asm/thread_info_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * thread_info.h: sparc low-level thread information * adapted from the ppc version by Pete Zaitcev, which was @@ -13,7 +14,7 @@ #ifdef __KERNEL__ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/ptrace.h> #include <asm/page.h> @@ -27,7 +28,6 @@ struct thread_info { unsigned long uwinmask; struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ int cpu; /* cpu we're on */ int preempt_count; /* 0 => preemptable, @@ -35,6 +35,8 @@ struct thread_info { int softirq_count; int hardirq_count; + u32 __unused; + /* Context switch saved kernel state. */ unsigned long ksp; /* ... ksp __attribute__ ((aligned (8))); */ unsigned long kpc; @@ -47,8 +49,6 @@ struct thread_info { struct reg_window32 reg_window[NSWINS]; /* align for ldd! */ unsigned long rwbuf_stkptrs[NSWINS]; unsigned long w_saved; - - struct restart_block restart_block; }; /* @@ -58,18 +58,11 @@ struct thread_info { { \ .uwinmask = 0, \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .restart_block = { \ - .fn = do_no_restart_syscall, \ - }, \ } -#define init_thread_info (init_thread_union.thread_info) -#define init_stack (init_thread_union.stack) - /* how to get the thread information struct from C */ register struct thread_info *current_thread_info_reg asm("g6"); #define current_thread_info() (current_thread_info_reg) @@ -79,7 +72,7 @@ register struct thread_info *current_thread_info_reg asm("g6"); */ #define THREAD_SIZE_ORDER 1 -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ /* Size of kernel stack for each process */ #define THREAD_SIZE (2 * PAGE_SIZE) @@ -90,12 +83,11 @@ register struct thread_info *current_thread_info_reg asm("g6"); */ #define TI_UWINMASK 0x00 /* uwinmask */ #define TI_TASK 0x04 -#define TI_EXECDOMAIN 0x08 /* exec_domain */ -#define TI_FLAGS 0x0c -#define TI_CPU 0x10 -#define TI_PREEMPT 0x14 /* preempt_count */ -#define TI_SOFTIRQ 0x18 /* softirq_count */ -#define TI_HARDIRQ 0x1c /* hardirq_count */ +#define TI_FLAGS 0x08 +#define TI_CPU 0x0c +#define TI_PREEMPT 0x10 /* preempt_count */ +#define TI_SOFTIRQ 0x14 /* softirq_count */ +#define TI_HARDIRQ 0x18 /* hardirq_count */ #define TI_KSP 0x20 /* ksp */ #define TI_KPC 0x24 /* kpc (ldd'ed with kpc) */ #define TI_KPSR 0x28 /* kpsr */ @@ -103,9 +95,6 @@ register struct thread_info *current_thread_info_reg asm("g6"); #define TI_REG_WINDOW 0x30 #define TI_RWIN_SPTRS 0x230 #define TI_W_SAVED 0x250 -/* #define TI_RESTART_BLOCK 0x25n */ /* Nobody cares */ - -#define PREEMPT_ACTIVE 0x4000000 /* * thread information flag bit numbers @@ -115,6 +104,7 @@ register struct thread_info *current_thread_info_reg asm("g6"); #define TIF_SIGPENDING 2 /* signal pending */ #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ #define TIF_RESTORE_SIGMASK 4 /* restore signal mask in do_signal() */ +#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ #define TIF_USEDFPU 8 /* FPU was used by this task * this quantum (SMP) */ #define TIF_POLLING_NRFLAG 9 /* true if poll_idle() is polling @@ -126,11 +116,14 @@ register struct thread_info *current_thread_info_reg asm("g6"); #define _TIF_NOTIFY_RESUME (1<<TIF_NOTIFY_RESUME) #define _TIF_SIGPENDING (1<<TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) +#define _TIF_NOTIFY_SIGNAL (1<<TIF_NOTIFY_SIGNAL) #define _TIF_USEDFPU (1<<TIF_USEDFPU) #define _TIF_POLLING_NRFLAG (1<<TIF_POLLING_NRFLAG) #define _TIF_DO_NOTIFY_RESUME_MASK (_TIF_NOTIFY_RESUME | \ - _TIF_SIGPENDING) + _TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL) + +#define is_32bit_task() (1) #endif /* __KERNEL__ */ diff --git a/arch/sparc/include/asm/thread_info_64.h b/arch/sparc/include/asm/thread_info_64.h index d5e504251079..c8a73dff27f8 100644 --- a/arch/sparc/include/asm/thread_info_64.h +++ b/arch/sparc/include/asm/thread_info_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* thread_info.h: sparc64 low-level thread information * * Copyright (C) 2002 David S. Miller (davem@redhat.com) @@ -25,13 +26,12 @@ #include <asm/page.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/ptrace.h> #include <asm/types.h> struct task_struct; -struct exec_domain; struct thread_info { /* D$ line 1 */ @@ -44,10 +44,9 @@ struct thread_info { /* D$ line 2 */ unsigned long fault_address; struct pt_regs *kregs; - struct exec_domain *exec_domain; int preempt_count; /* 0 => preemptable, <0 => BUG */ __u8 new_child; - __u8 current_ds; + __u8 __pad; __u16 cpu; unsigned long *utraps; @@ -58,15 +57,14 @@ struct thread_info { unsigned long gsr[7]; unsigned long xfsr[7]; - struct restart_block restart_block; - struct pt_regs *kern_una_regs; unsigned int kern_una_insn; - unsigned long fpregs[0] __attribute__ ((aligned(64))); + unsigned long fpregs[(7 * 256) / sizeof(unsigned long)] + __attribute__ ((aligned(64))); }; -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ /* offsets into the thread_info struct for assembly code access */ #define TI_TASK 0x00000000 @@ -81,20 +79,17 @@ struct thread_info { #define TI_KSP 0x00000018 #define TI_FAULT_ADDR 0x00000020 #define TI_KREGS 0x00000028 -#define TI_EXEC_DOMAIN 0x00000030 -#define TI_PRE_COUNT 0x00000038 -#define TI_NEW_CHILD 0x0000003c -#define TI_CURRENT_DS 0x0000003d -#define TI_CPU 0x0000003e -#define TI_UTRAPS 0x00000040 -#define TI_REG_WINDOW 0x00000048 -#define TI_RWIN_SPTRS 0x000003c8 -#define TI_GSR 0x00000400 -#define TI_XFSR 0x00000438 -#define TI_RESTART_BLOCK 0x00000470 -#define TI_KUNA_REGS 0x000004a0 -#define TI_KUNA_INSN 0x000004a8 -#define TI_FPREGS 0x000004c0 +#define TI_PRE_COUNT 0x00000030 +#define TI_NEW_CHILD 0x00000034 +#define TI_CPU 0x00000036 +#define TI_UTRAPS 0x00000038 +#define TI_REG_WINDOW 0x00000040 +#define TI_RWIN_SPTRS 0x000003c0 +#define TI_GSR 0x000003f8 +#define TI_XFSR 0x00000430 +#define TI_KUNA_REGS 0x00000468 +#define TI_KUNA_INSN 0x00000470 +#define TI_FPREGS 0x00000480 /* We embed this in the uppermost byte of thread_info->flags */ #define FAULT_CODE_WRITE 0x01 /* Write access, implies D-TLB */ @@ -102,6 +97,7 @@ struct thread_info { #define FAULT_CODE_ITLB 0x04 /* Miss happened in I-TLB */ #define FAULT_CODE_WINFIXUP 0x08 /* Miss happened during spill/fill */ #define FAULT_CODE_BLKCOMMIT 0x10 /* Use blk-commit ASI in copy_page */ +#define FAULT_CODE_BAD_RA 0x20 /* Bad RA for sun4v */ #if PAGE_SHIFT == 13 #define THREAD_SIZE (2*PAGE_SIZE) @@ -111,30 +107,25 @@ struct thread_info { #define THREAD_SHIFT PAGE_SHIFT #endif /* PAGE_SHIFT == 13 */ -#define PREEMPT_ACTIVE 0x10000000 - /* * macros/functions for gaining access to the thread information structure */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .current_ds = ASI_P, \ - .exec_domain = &default_exec_domain, \ .preempt_count = INIT_PREEMPT_COUNT, \ - .restart_block = { \ - .fn = do_no_restart_syscall, \ - }, \ + .kregs = (struct pt_regs *)(init_stack+THREAD_SIZE)-1 \ } -#define init_thread_info (init_thread_union.thread_info) -#define init_stack (init_thread_union.stack) - /* how to get the thread information struct from C */ +#ifndef BUILD_VDSO register struct thread_info *current_thread_info_reg asm("g6"); #define current_thread_info() (current_thread_info_reg) +#else +extern struct thread_info *current_thread_info(void); +#endif /* thread information allocation */ #if PAGE_SHIFT == 13 @@ -159,7 +150,7 @@ register struct thread_info *current_thread_info_reg asm("g6"); #define set_thread_fpdepth(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_FPDEPTH] = (val)) #define get_thread_wsaved() (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_WSAVED]) #define set_thread_wsaved(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_WSAVED] = (val)) -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ /* * Thread information flags, only 16 bits are available as we encode @@ -188,11 +179,11 @@ register struct thread_info *current_thread_info_reg asm("g6"); #define TIF_NOTIFY_RESUME 1 /* callback before returning to user */ #define TIF_SIGPENDING 2 /* signal pending */ #define TIF_NEED_RESCHED 3 /* rescheduling necessary */ -/* flag bit 4 is available */ +#define TIF_NOTIFY_SIGNAL 4 /* signal notifications exist */ #define TIF_UNALIGNED 5 /* allowed to do unaligned accesses */ -/* flag bit 6 is available */ +#define TIF_UPROBE 6 /* breakpointed or singlestepped */ #define TIF_32BIT 7 /* 32-bit binary */ -/* flag bit 8 is available */ +#define TIF_NOHZ 8 /* in adaptive nohz mode */ #define TIF_SECCOMP 9 /* secure computing */ #define TIF_SYSCALL_AUDIT 10 /* syscall auditing active */ #define TIF_SYSCALL_TRACEPOINT 11 /* syscall tracepoint instrumentation */ @@ -200,7 +191,7 @@ register struct thread_info *current_thread_info_reg asm("g6"); * in using in assembly, else we can't use the mask as * an immediate value in instructions such as andcc. */ -/* flag bit 12 is available */ +#define TIF_MCDPER 12 /* Precise MCD exception */ #define TIF_MEMDIE 13 /* is terminating due to OOM killer */ #define TIF_POLLING_NRFLAG 14 @@ -208,8 +199,11 @@ register struct thread_info *current_thread_info_reg asm("g6"); #define _TIF_NOTIFY_RESUME (1<<TIF_NOTIFY_RESUME) #define _TIF_SIGPENDING (1<<TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) +#define _TIF_NOTIFY_SIGNAL (1<<TIF_NOTIFY_SIGNAL) #define _TIF_UNALIGNED (1<<TIF_UNALIGNED) +#define _TIF_UPROBE (1<<TIF_UPROBE) #define _TIF_32BIT (1<<TIF_32BIT) +#define _TIF_NOHZ (1<<TIF_NOHZ) #define _TIF_SECCOMP (1<<TIF_SECCOMP) #define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT) #define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT) @@ -218,7 +212,11 @@ register struct thread_info *current_thread_info_reg asm("g6"); #define _TIF_USER_WORK_MASK ((0xff << TI_FLAG_WSAVED_SHIFT) | \ _TIF_DO_NOTIFY_RESUME_MASK | \ _TIF_NEED_RESCHED) -#define _TIF_DO_NOTIFY_RESUME_MASK (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING) +#define _TIF_DO_NOTIFY_RESUME_MASK (_TIF_NOTIFY_RESUME | \ + _TIF_SIGPENDING | _TIF_UPROBE | \ + _TIF_NOTIFY_SIGNAL) + +#define is_32bit_task() (test_thread_flag(TIF_32BIT)) /* * Thread-synchronous status. @@ -229,39 +227,15 @@ register struct thread_info *current_thread_info_reg asm("g6"); * * Note that there are only 8 bits available. */ -#define TS_RESTORE_SIGMASK 0x0001 /* restore signal mask in do_signal() */ - -#ifndef __ASSEMBLY__ -#define HAVE_SET_RESTORE_SIGMASK 1 -static inline void set_restore_sigmask(void) -{ - struct thread_info *ti = current_thread_info(); - ti->status |= TS_RESTORE_SIGMASK; - WARN_ON(!test_bit(TIF_SIGPENDING, &ti->flags)); -} -static inline void clear_restore_sigmask(void) -{ - current_thread_info()->status &= ~TS_RESTORE_SIGMASK; -} -static inline bool test_restore_sigmask(void) -{ - return current_thread_info()->status & TS_RESTORE_SIGMASK; -} -static inline bool test_and_clear_restore_sigmask(void) -{ - struct thread_info *ti = current_thread_info(); - if (!(ti->status & TS_RESTORE_SIGMASK)) - return false; - ti->status &= ~TS_RESTORE_SIGMASK; - return true; -} + +#ifndef __ASSEMBLER__ #define thread32_stack_is_64bit(__SP) (((__SP) & 0x1) != 0) #define test_thread_64bit_stack(__SP) \ ((test_thread_flag(TIF_32BIT) && !thread32_stack_is_64bit(__SP)) ? \ false : true) -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* __KERNEL__ */ diff --git a/arch/sparc/include/asm/timer.h b/arch/sparc/include/asm/timer.h index 612fd2779d9e..eaf6195d17e2 100644 --- a/arch/sparc/include/asm/timer.h +++ b/arch/sparc/include/asm/timer.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_TIMER_H #define ___ASM_SPARC_TIMER_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/timer_32.h b/arch/sparc/include/asm/timer_32.h index 72f40a546de3..eecd2696922d 100644 --- a/arch/sparc/include/asm/timer_32.h +++ b/arch/sparc/include/asm/timer_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * timer.h: Definitions for the timer chips on the Sparc. * @@ -32,13 +33,13 @@ static inline unsigned int timer_value(unsigned int value) return (value + 1) << TIMER_VALUE_SHIFT; } -extern __volatile__ unsigned int *master_l10_counter; +extern volatile u32 __iomem *master_l10_counter; -extern irqreturn_t notrace timer_interrupt(int dummy, void *dev_id); +irqreturn_t notrace timer_interrupt(int dummy, void *dev_id); #ifdef CONFIG_SMP DECLARE_PER_CPU(struct clock_event_device, sparc32_clockevent); -extern void register_percpu_ce(int cpu); +void register_percpu_ce(int cpu); #endif #endif /* !(_SPARC_TIMER_H) */ diff --git a/arch/sparc/include/asm/timer_64.h b/arch/sparc/include/asm/timer_64.h index 01197d8215c4..ffff52c8b760 100644 --- a/arch/sparc/include/asm/timer_64.h +++ b/arch/sparc/include/asm/timer_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* timer.h: System timer definitions for sun5. * * Copyright (C) 1997, 2008 David S. Miller (davem@davemloft.net) @@ -6,10 +7,16 @@ #ifndef _SPARC64_TIMER_H #define _SPARC64_TIMER_H +#include <uapi/asm/asi.h> #include <linux/types.h> #include <linux/init.h> +/* The most frequently accessed fields should be first, + * to fit into the same cacheline. + */ struct sparc64_tick_ops { + unsigned long ticks_per_nsec_quotient; + unsigned long offset; unsigned long long (*get_tick)(void); int (*add_compare)(unsigned long); unsigned long softint_mask; @@ -17,14 +24,75 @@ struct sparc64_tick_ops { void (*init_tick)(void); unsigned long (*add_tick)(unsigned long); + unsigned long (*get_frequency)(void); + unsigned long frequency; char *name; }; extern struct sparc64_tick_ops *tick_ops; -extern unsigned long sparc64_get_clock_tick(unsigned int cpu); -extern void setup_sparc64_timer(void); -extern void __init time_init(void); +unsigned long sparc64_get_clock_tick(unsigned int cpu); +void setup_sparc64_timer(void); + +#define TICK_PRIV_BIT BIT(63) +#define TICKCMP_IRQ_BIT BIT(63) + +#define HBIRD_STICKCMP_ADDR 0x1fe0000f060UL +#define HBIRD_STICK_ADDR 0x1fe0000f070UL + +#define GET_TICK_NINSTR 13 +struct get_tick_patch { + unsigned int addr; + unsigned int tick[GET_TICK_NINSTR]; + unsigned int stick[GET_TICK_NINSTR]; +}; + +extern struct get_tick_patch __get_tick_patch; +extern struct get_tick_patch __get_tick_patch_end; + +static inline unsigned long get_tick(void) +{ + unsigned long tick, tmp1, tmp2; + + __asm__ __volatile__( + /* read hbtick 13 instructions */ + "661:\n" + " mov 0x1fe, %1\n" + " sllx %1, 0x20, %1\n" + " sethi %%hi(0xf000), %2\n" + " or %2, 0x70, %2\n" + " or %1, %2, %1\n" /* %1 = HBIRD_STICK_ADDR */ + " add %1, 8, %2\n" + " ldxa [%2]%3, %0\n" + " ldxa [%1]%3, %1\n" + " ldxa [%2]%3, %2\n" + " sub %2, %0, %0\n" /* don't modify %xcc */ + " brnz,pn %0, 661b\n" /* restart to save one register */ + " sllx %2, 32, %2\n" + " or %2, %1, %0\n" + /* Common/not patched code */ + " sllx %0, 1, %0\n" + " srlx %0, 1, %0\n" /* Clear TICK_PRIV_BIT */ + /* Beginning of patch section */ + " .section .get_tick_patch, \"ax\"\n" + " .word 661b\n" + /* read tick 2 instructions and 11 skipped */ + " ba 1f\n" + " rd %%tick, %0\n" + " .skip 4 * (%4 - 2)\n" + "1:\n" + /* read stick 2 instructions and 11 skipped */ + " ba 1f\n" + " rd %%asr24, %0\n" + " .skip 4 * (%4 - 2)\n" + "1:\n" + /* End of patch section */ + " .previous\n" + : "=&r" (tick), "=&r" (tmp1), "=&r" (tmp2) + : "i" (ASI_PHYS_BYPASS_EC_E), "i" (GET_TICK_NINSTR)); + + return tick; +} #endif /* _SPARC64_TIMER_H */ diff --git a/arch/sparc/include/asm/timex.h b/arch/sparc/include/asm/timex.h index 70cc37b73827..9aac26b1f45f 100644 --- a/arch/sparc/include/asm/timex.h +++ b/arch/sparc/include/asm/timex.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_TIMEX_H #define ___ASM_SPARC_TIMEX_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/timex_32.h b/arch/sparc/include/asm/timex_32.h index b6ccdb0d6f7d..f86326a6f89e 100644 --- a/arch/sparc/include/asm/timex_32.h +++ b/arch/sparc/include/asm/timex_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * linux/include/asm/timex.h * @@ -8,8 +9,6 @@ #define CLOCK_TICK_RATE 1193180 /* Underlying HZ */ -/* XXX Maybe do something better at some point... -DaveM */ -typedef unsigned long cycles_t; -#define get_cycles() (0) +#include <asm-generic/timex.h> #endif diff --git a/arch/sparc/include/asm/timex_64.h b/arch/sparc/include/asm/timex_64.h index 18b30bc9823b..076c44f6845d 100644 --- a/arch/sparc/include/asm/timex_64.h +++ b/arch/sparc/include/asm/timex_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * linux/include/asm/timex.h * diff --git a/arch/sparc/include/asm/tlb.h b/arch/sparc/include/asm/tlb.h index 92d0393bbcdc..7146a577257f 100644 --- a/arch/sparc/include/asm/tlb.h +++ b/arch/sparc/include/asm/tlb.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_TLB_H #define ___ASM_SPARC_TLB_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/tlb_32.h b/arch/sparc/include/asm/tlb_32.h index 6d02d1ce53f3..5cd28a8793e3 100644 --- a/arch/sparc/include/asm/tlb_32.h +++ b/arch/sparc/include/asm/tlb_32.h @@ -1,24 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_TLB_H #define _SPARC_TLB_H -#define tlb_start_vma(tlb, vma) \ -do { \ - flush_cache_range(vma, vma->vm_start, vma->vm_end); \ -} while (0) - -#define tlb_end_vma(tlb, vma) \ -do { \ - flush_tlb_range(vma, vma->vm_start, vma->vm_end); \ -} while (0) - -#define __tlb_remove_tlb_entry(tlb, pte, address) \ - do { } while (0) - -#define tlb_flush(tlb) \ -do { \ - flush_tlb_mm((tlb)->mm); \ -} while (0) - #include <asm-generic/tlb.h> #endif /* _SPARC_TLB_H */ diff --git a/arch/sparc/include/asm/tlb_64.h b/arch/sparc/include/asm/tlb_64.h index 190e18913cc6..1a6e694418e3 100644 --- a/arch/sparc/include/asm/tlb_64.h +++ b/arch/sparc/include/asm/tlb_64.h @@ -1,32 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_TLB_H #define _SPARC64_TLB_H #include <linux/swap.h> #include <linux/pagemap.h> -#include <asm/pgalloc.h> #include <asm/tlbflush.h> #include <asm/mmu_context.h> #ifdef CONFIG_SMP -extern void smp_flush_tlb_pending(struct mm_struct *, +void smp_flush_tlb_pending(struct mm_struct *, unsigned long, unsigned long *); #endif #ifdef CONFIG_SMP -extern void smp_flush_tlb_mm(struct mm_struct *mm); +void smp_flush_tlb_mm(struct mm_struct *mm); #define do_flush_tlb_mm(mm) smp_flush_tlb_mm(mm) #else #define do_flush_tlb_mm(mm) __flush_tlb_mm(CTX_HWBITS(mm->context), SECONDARY_CONTEXT) #endif -extern void __flush_tlb_pending(unsigned long, unsigned long, unsigned long *); -extern void flush_tlb_pending(void); +void __flush_tlb_pending(unsigned long, unsigned long, unsigned long *); +void flush_tlb_pending(void); -#define tlb_start_vma(tlb, vma) do { } while (0) -#define tlb_end_vma(tlb, vma) do { } while (0) -#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) #define tlb_flush(tlb) flush_tlb_pending() +/* + * SPARC64's hardware TLB fill does not use the Linux page-tables + * and therefore we don't need a TLBI when freeing page-table pages. + */ + +#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE +#define tlb_needs_table_invalidate() (false) +#endif + +#define __HAVE_ARCH_TLB_REMOVE_TABLE #include <asm-generic/tlb.h> #endif /* _SPARC64_TLB_H */ diff --git a/arch/sparc/include/asm/tlbflush.h b/arch/sparc/include/asm/tlbflush.h index 2c9629fad1e2..30ae3ea21054 100644 --- a/arch/sparc/include/asm/tlbflush.h +++ b/arch/sparc/include/asm/tlbflush.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_TLBFLUSH_H #define ___ASM_SPARC_TLBFLUSH_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/tlbflush_32.h b/arch/sparc/include/asm/tlbflush_32.h index a5c4142130f5..470531991a08 100644 --- a/arch/sparc/include/asm/tlbflush_32.h +++ b/arch/sparc/include/asm/tlbflush_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_TLBFLUSH_H #define _SPARC_TLBFLUSH_H diff --git a/arch/sparc/include/asm/tlbflush_64.h b/arch/sparc/include/asm/tlbflush_64.h index f0d6a9700f4c..8b8cdaa69272 100644 --- a/arch/sparc/include/asm/tlbflush_64.h +++ b/arch/sparc/include/asm/tlbflush_64.h @@ -1,7 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_TLBFLUSH_H #define _SPARC64_TLBFLUSH_H -#include <linux/mm.h> #include <asm/mmu_context.h> /* TSB flush operations. */ @@ -9,15 +9,17 @@ #define TLB_BATCH_NR 192 struct tlb_batch { + unsigned int hugepage_shift; struct mm_struct *mm; unsigned long tlb_nr; unsigned long active; unsigned long vaddrs[TLB_BATCH_NR]; }; -extern void flush_tsb_kernel_range(unsigned long start, unsigned long end); -extern void flush_tsb_user(struct tlb_batch *tb); -extern void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr); +void flush_tsb_kernel_range(unsigned long start, unsigned long end); +void flush_tsb_user(struct tlb_batch *tb); +void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, + unsigned int hugepage_shift); /* TLB flush operations. */ @@ -35,25 +37,22 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, { } +void flush_tlb_kernel_range(unsigned long start, unsigned long end); + #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE -extern void flush_tlb_pending(void); -extern void arch_enter_lazy_mmu_mode(void); -extern void arch_leave_lazy_mmu_mode(void); +void flush_tlb_pending(void); +void arch_enter_lazy_mmu_mode(void); +void arch_leave_lazy_mmu_mode(void); #define arch_flush_lazy_mmu_mode() do {} while (0) /* Local cpu only. */ -extern void __flush_tlb_all(void); -extern void __flush_tlb_page(unsigned long context, unsigned long vaddr); -extern void __flush_tlb_kernel_range(unsigned long start, unsigned long end); +void __flush_tlb_all(void); +void __flush_tlb_page(unsigned long context, unsigned long vaddr); +void __flush_tlb_kernel_range(unsigned long start, unsigned long end); #ifndef CONFIG_SMP -#define flush_tlb_kernel_range(start,end) \ -do { flush_tsb_kernel_range(start,end); \ - __flush_tlb_kernel_range(start,end); \ -} while (0) - static inline void global_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr) { __flush_tlb_page(CTX_HWBITS(mm->context), vaddr); @@ -61,13 +60,8 @@ static inline void global_flush_tlb_page(struct mm_struct *mm, unsigned long vad #else /* CONFIG_SMP */ -extern void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end); -extern void smp_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr); - -#define flush_tlb_kernel_range(start, end) \ -do { flush_tsb_kernel_range(start,end); \ - smp_flush_tlb_kernel_range(start, end); \ -} while (0) +void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end); +void smp_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr); #define global_flush_tlb_page(mm, vaddr) \ smp_flush_tlb_page(mm, vaddr) diff --git a/arch/sparc/include/asm/topology.h b/arch/sparc/include/asm/topology.h index ee4f191d394a..ba7b9d9d91cf 100644 --- a/arch/sparc/include/asm/topology.h +++ b/arch/sparc/include/asm/topology.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_TOPOLOGY_H #define ___ASM_SPARC_TOPOLOGY_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/topology_32.h b/arch/sparc/include/asm/topology_32.h index ee5ac9c9da28..66c4f9ff700d 100644 --- a/arch/sparc/include/asm/topology_32.h +++ b/arch/sparc/include/asm/topology_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_SPARC_TOPOLOGY_H #define _ASM_SPARC_TOPOLOGY_H diff --git a/arch/sparc/include/asm/topology_64.h b/arch/sparc/include/asm/topology_64.h index 1754390a426f..34c628a22ea5 100644 --- a/arch/sparc/include/asm/topology_64.h +++ b/arch/sparc/include/asm/topology_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_SPARC64_TOPOLOGY_H #define _ASM_SPARC64_TOPOLOGY_H @@ -10,15 +11,13 @@ static inline int cpu_to_node(int cpu) return numa_cpu_lookup_table[cpu]; } -#define parent_node(node) (node) - #define cpumask_of_node(node) ((node) == -1 ? \ cpu_all_mask : \ &numa_cpumask_lookup_table[node]) struct pci_bus; #ifdef CONFIG_PCI -extern int pcibus_to_node(struct pci_bus *pbus); +int pcibus_to_node(struct pci_bus *pbus); #else static inline int pcibus_to_node(struct pci_bus *pbus) { @@ -31,6 +30,9 @@ static inline int pcibus_to_node(struct pci_bus *pbus) cpu_all_mask : \ cpumask_of_node(pcibus_to_node(bus))) +int __node_distance(int, int); +#define node_distance(a, b) __node_distance(a, b) + #else /* CONFIG_NUMA */ #include <asm-generic/topology.h> @@ -38,18 +40,26 @@ static inline int pcibus_to_node(struct pci_bus *pbus) #endif /* !(CONFIG_NUMA) */ #ifdef CONFIG_SMP + +#include <asm/cpudata.h> + #define topology_physical_package_id(cpu) (cpu_data(cpu).proc_id) #define topology_core_id(cpu) (cpu_data(cpu).core_id) -#define topology_core_cpumask(cpu) (&cpu_core_map[cpu]) -#define topology_thread_cpumask(cpu) (&per_cpu(cpu_sibling_map, cpu)) -#define mc_capable() (sparc64_multi_core) -#define smt_capable() (sparc64_multi_core) +#define topology_core_cpumask(cpu) (&cpu_core_sib_map[cpu]) +#define topology_core_cache_cpumask(cpu) (&cpu_core_sib_cache_map[cpu]) +#define topology_sibling_cpumask(cpu) (&per_cpu(cpu_sibling_map, cpu)) #endif /* CONFIG_SMP */ extern cpumask_t cpu_core_map[NR_CPUS]; +extern cpumask_t cpu_core_sib_map[NR_CPUS]; +extern cpumask_t cpu_core_sib_cache_map[NR_CPUS]; + +/** + * Return cores that shares the last level cache. + */ static inline const struct cpumask *cpu_coregroup_mask(int cpu) { - return &cpu_core_map[cpu]; + return &cpu_core_sib_cache_map[cpu]; } #endif /* _ASM_SPARC64_TOPOLOGY_H */ diff --git a/arch/sparc/include/asm/trap_block.h b/arch/sparc/include/asm/trap_block.h index 7e26b2db6211..6cf2a60a0156 100644 --- a/arch/sparc/include/asm/trap_block.h +++ b/arch/sparc/include/asm/trap_block.h @@ -1,10 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_TRAP_BLOCK_H #define _SPARC_TRAP_BLOCK_H +#include <linux/threads.h> + #include <asm/hypervisor.h> #include <asm/asi.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* Trap handling code needs to get at a few critical values upon * trap entry and to process TSB misses. These cannot be in the @@ -51,11 +54,12 @@ struct trap_per_cpu { unsigned long __per_cpu_base; } __attribute__((aligned(64))); extern struct trap_per_cpu trap_block[NR_CPUS]; -extern void init_cur_cpu_trap(struct thread_info *); -extern void setup_tba(void); +void init_cur_cpu_trap(struct thread_info *); +void setup_tba(void); extern int ncpus_probed; +extern u64 cpu_mondo_counter[NR_CPUS]; -extern unsigned long real_hard_smp_processor_id(void); +unsigned long real_hard_smp_processor_id(void); struct cpuid_patch_entry { unsigned int addr; @@ -72,6 +76,10 @@ struct sun4v_1insn_patch_entry { }; extern struct sun4v_1insn_patch_entry __sun4v_1insn_patch, __sun4v_1insn_patch_end; +extern struct sun4v_1insn_patch_entry __fast_win_ctrl_1insn_patch, + __fast_win_ctrl_1insn_patch_end; +extern struct sun4v_1insn_patch_entry __sun_m7_1insn_patch, + __sun_m7_1insn_patch_end; struct sun4v_2insn_patch_entry { unsigned int addr; @@ -79,9 +87,11 @@ struct sun4v_2insn_patch_entry { }; extern struct sun4v_2insn_patch_entry __sun4v_2insn_patch, __sun4v_2insn_patch_end; +extern struct sun4v_2insn_patch_entry __sun_m7_2insn_patch, + __sun_m7_2insn_patch_end; -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #define TRAP_PER_CPU_THREAD 0x00 #define TRAP_PER_CPU_PGD_PADDR 0x08 diff --git a/arch/sparc/include/asm/traps.h b/arch/sparc/include/asm/traps.h index 51abcb1f9b3b..e4e10b0e7887 100644 --- a/arch/sparc/include/asm/traps.h +++ b/arch/sparc/include/asm/traps.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * traps.h: Format of entries for the Sparc trap table. * @@ -8,7 +9,7 @@ #include <uapi/asm/traps.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* This is for V8 compliant Sparc CPUS */ struct tt_entry { unsigned long inst_one; @@ -20,5 +21,5 @@ struct tt_entry { /* We set this to _start in system setup. */ extern struct tt_entry *sparc_ttable; -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #endif /* !(_SPARC_TRAPS_H) */ diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h index e696432b950d..239be259e166 100644 --- a/arch/sparc/include/asm/tsb.h +++ b/arch/sparc/include/asm/tsb.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_TSB_H #define _SPARC64_TSB_H @@ -58,7 +59,7 @@ * The kernel TSB is locked into the TLB by virtue of being in the * kernel image, so we don't play these games for swapper_tsb access. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct tsb_ldquad_phys_patch_entry { unsigned int addr; unsigned int sun4u_insn; @@ -133,107 +134,124 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; sub TSB, 0x8, TSB; \ TSB_STORE(TSB, TAG); - /* Do a kernel page table walk. Leaves physical PTE pointer in - * REG1. Jumps to FAIL_LABEL on early page table walk termination. - * VADDR will not be clobbered, but REG2 will. + /* Do a kernel page table walk. Leaves valid PTE value in + * REG1. Jumps to FAIL_LABEL on early page table walk + * termination. VADDR will not be clobbered, but REG2 will. + * + * There are two masks we must apply to propagate bits from + * the virtual address into the PTE physical address field + * when dealing with huge pages. This is because the page + * table boundaries do not match the huge page size(s) the + * hardware supports. + * + * In these cases we propagate the bits that are below the + * page table level where we saw the huge page mapping, but + * are still within the relevant physical bits for the huge + * page size in question. So for PMD mappings (which fall on + * bit 23, for 8MB per PMD) we must propagate bit 22 for a + * 4MB huge page. For huge PUDs (which fall on bit 33, for + * 8GB per PUD), we have to accommodate 256MB and 2GB huge + * pages. So for those we propagate bits 32 to 28. */ #define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL) \ sethi %hi(swapper_pg_dir), REG1; \ or REG1, %lo(swapper_pg_dir), REG1; \ sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \ srlx REG2, 64 - PAGE_SHIFT, REG2; \ - andn REG2, 0x3, REG2; \ - lduw [REG1 + REG2], REG1; \ + andn REG2, 0x7, REG2; \ + ldx [REG1 + REG2], REG1; \ brz,pn REG1, FAIL_LABEL; \ - sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ + sllx VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \ + srlx REG2, 64 - PAGE_SHIFT, REG2; \ + andn REG2, 0x7, REG2; \ + ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ + brz,pn REG1, FAIL_LABEL; \ + sethi %uhi(_PAGE_PUD_HUGE), REG2; \ + brz,pn REG1, FAIL_LABEL; \ + sllx REG2, 32, REG2; \ + andcc REG1, REG2, %g0; \ + sethi %hi(0xf8000000), REG2; \ + bne,pt %xcc, 697f; \ + sllx REG2, 1, REG2; \ + sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ srlx REG2, 64 - PAGE_SHIFT, REG2; \ - sllx REG1, PGD_PADDR_SHIFT, REG1; \ - andn REG2, 0x3, REG2; \ - lduwa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ + andn REG2, 0x7, REG2; \ + ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ + sethi %uhi(_PAGE_PMD_HUGE), REG2; \ brz,pn REG1, FAIL_LABEL; \ - sllx VADDR, 64 - PMD_SHIFT, REG2; \ - srlx REG2, 64 - (PAGE_SHIFT - 1), REG2; \ - sllx REG1, PMD_PADDR_SHIFT, REG1; \ + sllx REG2, 32, REG2; \ + andcc REG1, REG2, %g0; \ + be,pn %xcc, 698f; \ + sethi %hi(0x400000), REG2; \ +697: brgez,pn REG1, FAIL_LABEL; \ + andn REG1, REG2, REG1; \ + and VADDR, REG2, REG2; \ + ba,pt %xcc, 699f; \ + or REG1, REG2, REG1; \ +698: sllx VADDR, 64 - PMD_SHIFT, REG2; \ + srlx REG2, 64 - PAGE_SHIFT, REG2; \ andn REG2, 0x7, REG2; \ - add REG1, REG2, REG1; + ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ + brgez,pn REG1, FAIL_LABEL; \ + nop; \ +699: - /* These macros exists only to make the PMD translator below - * easier to read. It hides the ELF section switch for the - * sun4v code patching. + /* PUD has been loaded into REG1, interpret the value, seeing + * if it is a HUGE PUD or a normal one. If it is not valid + * then jump to FAIL_LABEL. If it is a HUGE PUD, and it + * translates to a valid PTE, branch to PTE_LABEL. + * + * We have to propagate bits [32:22] from the virtual address + * to resolve at 4M granularity. */ -#define OR_PTE_BIT_1INSN(REG, NAME) \ -661: or REG, _PAGE_##NAME##_4U, REG; \ - .section .sun4v_1insn_patch, "ax"; \ - .word 661b; \ - or REG, _PAGE_##NAME##_4V, REG; \ - .previous; - -#define OR_PTE_BIT_2INSN(REG, TMP, NAME) \ -661: sethi %hi(_PAGE_##NAME##_4U), TMP; \ - or REG, TMP, REG; \ - .section .sun4v_2insn_patch, "ax"; \ - .word 661b; \ - mov -1, TMP; \ - or REG, _PAGE_##NAME##_4V, REG; \ - .previous; - - /* Load into REG the PTE value for VALID, CACHE, and SZHUGE. */ -#define BUILD_PTE_VALID_SZHUGE_CACHE(REG) \ -661: sethi %uhi(_PAGE_VALID|_PAGE_SZHUGE_4U), REG; \ - .section .sun4v_1insn_patch, "ax"; \ - .word 661b; \ - sethi %uhi(_PAGE_VALID), REG; \ - .previous; \ - sllx REG, 32, REG; \ -661: or REG, _PAGE_CP_4U|_PAGE_CV_4U, REG; \ - .section .sun4v_1insn_patch, "ax"; \ - .word 661b; \ - or REG, _PAGE_CP_4V|_PAGE_CV_4V|_PAGE_SZHUGE_4V, REG; \ - .previous; +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +#define USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ +700: ba 700f; \ + nop; \ + .section .pud_huge_patch, "ax"; \ + .word 700b; \ + nop; \ + .previous; \ + brz,pn REG1, FAIL_LABEL; \ + sethi %uhi(_PAGE_PUD_HUGE), REG2; \ + sllx REG2, 32, REG2; \ + andcc REG1, REG2, %g0; \ + be,pt %xcc, 700f; \ + sethi %hi(0xffe00000), REG2; \ + sllx REG2, 1, REG2; \ + brgez,pn REG1, FAIL_LABEL; \ + andn REG1, REG2, REG1; \ + and VADDR, REG2, REG2; \ + brlz,pt REG1, PTE_LABEL; \ + or REG1, REG2, REG1; \ +700: +#else +#define USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ + brz,pn REG1, FAIL_LABEL; \ + nop; +#endif /* PMD has been loaded into REG1, interpret the value, seeing * if it is a HUGE PMD or a normal one. If it is not valid * then jump to FAIL_LABEL. If it is a HUGE PMD, and it * translates to a valid PTE, branch to PTE_LABEL. * - * We translate the PMD by hand, one bit at a time, - * constructing the huge PTE. - * - * So we construct the PTE in REG2 as follows: - * - * 1) Extract the PMD PFN from REG1 and place it into REG2. - * - * 2) Translate PMD protection bits in REG1 into REG2, one bit - * at a time using andcc tests on REG1 and OR's into REG2. - * - * Only two bits to be concerned with here, EXEC and WRITE. - * Now REG1 is freed up and we can use it as a temporary. - * - * 3) Construct the VALID, CACHE, and page size PTE bits in - * REG1, OR with REG2 to form final PTE. + * We have to propagate the 4MB bit of the virtual address + * because we are fabricating 8MB pages using 4MB hw pages. */ -#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ - brz,pn REG1, FAIL_LABEL; \ - andcc REG1, PMD_ISHUGE, %g0; \ - be,pt %xcc, 700f; \ - and REG1, PMD_HUGE_PRESENT|PMD_HUGE_ACCESSED, REG2; \ - cmp REG2, PMD_HUGE_PRESENT|PMD_HUGE_ACCESSED; \ - bne,pn %xcc, FAIL_LABEL; \ - andn REG1, PMD_HUGE_PROTBITS, REG2; \ - sllx REG2, PMD_PADDR_SHIFT, REG2; \ - /* REG2 now holds PFN << PAGE_SHIFT */ \ - andcc REG1, PMD_HUGE_WRITE, %g0; \ - bne,a,pt %xcc, 1f; \ - OR_PTE_BIT_1INSN(REG2, W); \ -1: andcc REG1, PMD_HUGE_EXEC, %g0; \ - be,pt %xcc, 1f; \ - nop; \ - OR_PTE_BIT_2INSN(REG2, REG1, EXEC); \ - /* REG1 can now be clobbered, build final PTE */ \ -1: BUILD_PTE_VALID_SZHUGE_CACHE(REG1); \ - ba,pt %xcc, PTE_LABEL; \ - or REG1, REG2, REG1; \ + brz,pn REG1, FAIL_LABEL; \ + sethi %uhi(_PAGE_PMD_HUGE), REG2; \ + sllx REG2, 32, REG2; \ + andcc REG1, REG2, %g0; \ + be,pt %xcc, 700f; \ + sethi %hi(4 * 1024 * 1024), REG2; \ + brgez,pn REG1, FAIL_LABEL; \ + andn REG1, REG2, REG1; \ + and VADDR, REG2, REG2; \ + brlz,pt REG1, PTE_LABEL; \ + or REG1, REG2, REG1; \ 700: #else #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ @@ -253,18 +271,22 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; #define USER_PGTABLE_WALK_TL1(VADDR, PHYS_PGD, REG1, REG2, FAIL_LABEL) \ sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \ srlx REG2, 64 - PAGE_SHIFT, REG2; \ - andn REG2, 0x3, REG2; \ - lduwa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \ + andn REG2, 0x7, REG2; \ + ldxa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \ + brz,pn REG1, FAIL_LABEL; \ + sllx VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \ + srlx REG2, 64 - PAGE_SHIFT, REG2; \ + andn REG2, 0x7, REG2; \ + ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ + USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \ brz,pn REG1, FAIL_LABEL; \ sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ srlx REG2, 64 - PAGE_SHIFT, REG2; \ - sllx REG1, PGD_PADDR_SHIFT, REG1; \ - andn REG2, 0x3, REG2; \ - lduwa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ + andn REG2, 0x7, REG2; \ + ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \ sllx VADDR, 64 - PMD_SHIFT, REG2; \ - srlx REG2, 64 - (PAGE_SHIFT - 1), REG2; \ - sllx REG1, PMD_PADDR_SHIFT, REG1; \ + srlx REG2, 64 - PAGE_SHIFT, REG2; \ andn REG2, 0x7, REG2; \ add REG1, REG2, REG1; \ ldxa [REG1] ASI_PHYS_USE_EC, REG1; \ @@ -306,8 +328,6 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; (KERNEL_TSB_SIZE_BYTES / 16) #define KERNEL_TSB4M_NENTRIES 4096 -#define KTSB_PHYS_SHIFT 15 - /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL * on TSB hit. REG1, REG2, REG3, and REG4 are used as temporaries * and the found TTE will be left in REG1. REG3 and REG4 must @@ -316,17 +336,15 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; * VADDR and TAG will be preserved and not clobbered by this macro. */ #define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ -661: sethi %hi(swapper_tsb), REG1; \ - or REG1, %lo(swapper_tsb), REG1; \ +661: sethi %uhi(swapper_tsb), REG1; \ + sethi %hi(swapper_tsb), REG2; \ + or REG1, %ulo(swapper_tsb), REG1; \ + or REG2, %lo(swapper_tsb), REG2; \ .section .swapper_tsb_phys_patch, "ax"; \ .word 661b; \ .previous; \ -661: nop; \ - .section .tsb_ldquad_phys_patch, "ax"; \ - .word 661b; \ - sllx REG1, KTSB_PHYS_SHIFT, REG1; \ - sllx REG1, KTSB_PHYS_SHIFT, REG1; \ - .previous; \ + sllx REG1, 32, REG1; \ + or REG1, REG2, REG1; \ srlx VADDR, PAGE_SHIFT, REG2; \ and REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \ sllx REG2, 4, REG2; \ @@ -341,17 +359,15 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; * we can make use of that for the index computation. */ #define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ -661: sethi %hi(swapper_4m_tsb), REG1; \ - or REG1, %lo(swapper_4m_tsb), REG1; \ +661: sethi %uhi(swapper_4m_tsb), REG1; \ + sethi %hi(swapper_4m_tsb), REG2; \ + or REG1, %ulo(swapper_4m_tsb), REG1; \ + or REG2, %lo(swapper_4m_tsb), REG2; \ .section .swapper_4m_tsb_phys_patch, "ax"; \ .word 661b; \ .previous; \ -661: nop; \ - .section .tsb_ldquad_phys_patch, "ax"; \ - .word 661b; \ - sllx REG1, KTSB_PHYS_SHIFT, REG1; \ - sllx REG1, KTSB_PHYS_SHIFT, REG1; \ - .previous; \ + sllx REG1, 32, REG1; \ + or REG1, REG2, REG1; \ and TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \ sllx REG2, 4, REG2; \ add REG1, REG2, REG2; \ diff --git a/arch/sparc/include/asm/tsunami.h b/arch/sparc/include/asm/tsunami.h index 5bbd1d523baa..acaf014eff46 100644 --- a/arch/sparc/include/asm/tsunami.h +++ b/arch/sparc/include/asm/tsunami.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * tsunami.h: Module specific definitions for Tsunami V8 Sparcs * diff --git a/arch/sparc/include/asm/ttable.h b/arch/sparc/include/asm/ttable.h index 71b5a67522ab..b32d3068cce1 100644 --- a/arch/sparc/include/asm/ttable.h +++ b/arch/sparc/include/asm/ttable.h @@ -1,10 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_TTABLE_H #define _SPARC64_TTABLE_H #include <asm/utrap.h> #include <asm/pil.h> -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #include <asm/thread_info.h> #endif @@ -186,6 +187,12 @@ #define KPROBES_TRAP(lvl) TRAP_ARG(bad_trap, lvl) #endif +#ifdef CONFIG_UPROBES +#define UPROBES_TRAP(lvl) TRAP_ARG(uprobe_trap, lvl) +#else +#define UPROBES_TRAP(lvl) TRAP_ARG(bad_trap, lvl) +#endif + #ifdef CONFIG_KGDB #define KGDB_TRAP(lvl) TRAP_IRQ(kgdb_trap, lvl) #else @@ -212,6 +219,16 @@ nop; \ nop; +#define SUN4V_MCD_PRECISE \ + ldxa [%g0] ASI_SCRATCHPAD, %g2; \ + ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4; \ + ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5; \ + ba,pt %xcc, etrap; \ + rd %pc, %g7; \ + ba,pt %xcc, sun4v_mcd_detect_precise; \ + nop; \ + nop; + /* Before touching these macros, you owe it to yourself to go and * see how arch/sparc64/kernel/winfixup.S works... -DaveM * @@ -589,8 +606,8 @@ user_rtt_fill_64bit: \ restored; \ nop; nop; nop; nop; nop; nop; \ nop; nop; nop; nop; nop; \ - ba,a,pt %xcc, user_rtt_fill_fixup; \ - ba,a,pt %xcc, user_rtt_fill_fixup; \ + ba,a,pt %xcc, user_rtt_fill_fixup_dax; \ + ba,a,pt %xcc, user_rtt_fill_fixup_mna; \ ba,a,pt %xcc, user_rtt_fill_fixup; @@ -652,8 +669,8 @@ user_rtt_fill_32bit: \ restored; \ nop; nop; nop; nop; nop; \ nop; nop; nop; \ - ba,a,pt %xcc, user_rtt_fill_fixup; \ - ba,a,pt %xcc, user_rtt_fill_fixup; \ + ba,a,pt %xcc, user_rtt_fill_fixup_dax; \ + ba,a,pt %xcc, user_rtt_fill_fixup_mna; \ ba,a,pt %xcc, user_rtt_fill_fixup; diff --git a/arch/sparc/include/asm/turbosparc.h b/arch/sparc/include/asm/turbosparc.h index 17c73282db0a..5f73263b6ded 100644 --- a/arch/sparc/include/asm/turbosparc.h +++ b/arch/sparc/include/asm/turbosparc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * turbosparc.h: Defines specific to the TurboSparc module. * This is SRMMU stuff. @@ -56,7 +57,7 @@ #define TURBOSPARC_WTENABLE 0x00000020 /* Write thru for dcache */ #define TURBOSPARC_SNENABLE 0x40000000 /* DVMA snoop enable */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* Bits [13:5] select one of 512 instruction cache tags */ static inline void turbosparc_inv_insn_tag(unsigned long addr) @@ -120,6 +121,6 @@ static inline unsigned long turbosparc_get_ccreg(void) return regval; } -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* !(_SPARC_TURBOSPARC_H) */ diff --git a/arch/sparc/include/asm/uaccess.h b/arch/sparc/include/asm/uaccess.h index 0167d26d0d1d..ee75f69e3fcd 100644 --- a/arch/sparc/include/asm/uaccess.h +++ b/arch/sparc/include/asm/uaccess.h @@ -1,14 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_UACCESS_H #define ___ASM_SPARC_UACCESS_H + +#include <asm/extable.h> + #if defined(__sparc__) && defined(__arch64__) #include <asm/uaccess_64.h> #else #include <asm/uaccess_32.h> #endif -#define user_addr_max() \ - (segment_eq(get_fs(), USER_DS) ? TASK_SIZE : ~0UL) - -extern long strncpy_from_user(char *dest, const char __user *src, long count); +long strncpy_from_user(char *dest, const char __user *src, long count); #endif diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index 53a28dd59f59..43284b6ec46a 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * uaccess.h: User space memore access functions. * @@ -7,80 +8,11 @@ #ifndef _ASM_UACCESS_H #define _ASM_UACCESS_H -#ifdef __KERNEL__ #include <linux/compiler.h> -#include <linux/sched.h> #include <linux/string.h> -#include <linux/errno.h> -#endif - -#ifndef __ASSEMBLY__ #include <asm/processor.h> - -#define ARCH_HAS_SORT_EXTABLE -#define ARCH_HAS_SEARCH_EXTABLE - -/* Sparc is not segmented, however we need to be able to fool access_ok() - * when doing system calls from kernel mode legitimately. - * - * "For historical reasons, these macros are grossly misnamed." -Linus - */ - -#define KERNEL_DS ((mm_segment_t) { 0 }) -#define USER_DS ((mm_segment_t) { -1 }) - -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - -#define get_ds() (KERNEL_DS) -#define get_fs() (current->thread.current_ds) -#define set_fs(val) ((current->thread.current_ds) = (val)) - -#define segment_eq(a,b) ((a).seg == (b).seg) - -/* We have there a nice not-mapped page at PAGE_OFFSET - PAGE_SIZE, so that this test - * can be fairly lightweight. - * No one can read/write anything from userland in the kernel space by setting - * large size and address near to PAGE_OFFSET - a fault will break his intentions. - */ -#define __user_ok(addr, size) ({ (void)(size); (addr) < STACK_TOP; }) -#define __kernel_ok (segment_eq(get_fs(), KERNEL_DS)) -#define __access_ok(addr,size) (__user_ok((addr) & get_fs().seg,(size))) -#define access_ok(type, addr, size) \ - ({ (void)(type); __access_ok((unsigned long)(addr), size); }) - -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - * - * There is a special way how to put a range of potentially faulting - * insns (like twenty ldd/std's with now intervening other instructions) - * You specify address of first in insn and 0 in fixup and in the next - * exception_table_entry you specify last potentially faulting insn + 1 - * and in fixup the routine which should handle the fault. - * That fixup code will get - * (faulting_insn_address - first_insn_in_the_range_address)/4 - * in %g2 (ie. index of the faulting instruction in the range). - */ - -struct exception_table_entry -{ - unsigned long insn, fixup; -}; - -/* Returns 0 if exception not found and fixup otherwise. */ -extern unsigned long search_extables_range(unsigned long addr, unsigned long *g2); - -extern void __ret_efault(void); +#include <asm-generic/access_ok.h> /* Uh, these should become the main single-value transfer routines.. * They automatically use the right size if we just have the right @@ -91,203 +23,182 @@ extern void __ret_efault(void); * of a performance impact. Thus we have a few rather ugly macros here, * and hide all the ugliness from the user. */ -#define put_user(x,ptr) ({ \ -unsigned long __pu_addr = (unsigned long)(ptr); \ -__chk_user_ptr(ptr); \ -__put_user_check((__typeof__(*(ptr)))(x),__pu_addr,sizeof(*(ptr))); }) - -#define get_user(x,ptr) ({ \ -unsigned long __gu_addr = (unsigned long)(ptr); \ -__chk_user_ptr(ptr); \ -__get_user_check((x),__gu_addr,sizeof(*(ptr)),__typeof__(*(ptr))); }) +#define put_user(x, ptr) ({ \ + void __user *__pu_addr = (ptr); \ + __chk_user_ptr(ptr); \ + __put_user_check((__typeof__(*(ptr)))(x), __pu_addr, sizeof(*(ptr))); \ +}) + +#define get_user(x, ptr) ({ \ + const void __user *__gu_addr = (ptr); \ + __chk_user_ptr(ptr); \ + __get_user_check((x), __gu_addr, sizeof(*(ptr)), __typeof__(*(ptr))); \ +}) /* * The "__xxx" versions do not do address space checking, useful when * doing multiple accesses to the same area (the user has to do the * checks by hand with "access_ok()") */ -#define __put_user(x,ptr) __put_user_nocheck((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr))) -#define __get_user(x,ptr) __get_user_nocheck((x),(ptr),sizeof(*(ptr)),__typeof__(*(ptr))) +#define __put_user(x, ptr) \ + __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) +#define __get_user(x, ptr) \ + __get_user_nocheck((x), (ptr), sizeof(*(ptr)), __typeof__(*(ptr))) struct __large_struct { unsigned long buf[100]; }; #define __m(x) ((struct __large_struct __user *)(x)) -#define __put_user_check(x,addr,size) ({ \ -register int __pu_ret; \ -if (__access_ok(addr,size)) { \ -switch (size) { \ -case 1: __put_user_asm(x,b,addr,__pu_ret); break; \ -case 2: __put_user_asm(x,h,addr,__pu_ret); break; \ -case 4: __put_user_asm(x,,addr,__pu_ret); break; \ -case 8: __put_user_asm(x,d,addr,__pu_ret); break; \ -default: __pu_ret = __put_user_bad(); break; \ -} } else { __pu_ret = -EFAULT; } __pu_ret; }) - -#define __put_user_nocheck(x,addr,size) ({ \ -register int __pu_ret; \ -switch (size) { \ -case 1: __put_user_asm(x,b,addr,__pu_ret); break; \ -case 2: __put_user_asm(x,h,addr,__pu_ret); break; \ -case 4: __put_user_asm(x,,addr,__pu_ret); break; \ -case 8: __put_user_asm(x,d,addr,__pu_ret); break; \ -default: __pu_ret = __put_user_bad(); break; \ -} __pu_ret; }) - -#define __put_user_asm(x,size,addr,ret) \ +#define __put_user_check(x, addr, size) ({ \ + register int __pu_ret; \ + if (__access_ok(addr, size)) { \ + switch (size) { \ + case 1: \ + __put_user_asm(x, b, addr, __pu_ret); \ + break; \ + case 2: \ + __put_user_asm(x, h, addr, __pu_ret); \ + break; \ + case 4: \ + __put_user_asm(x, , addr, __pu_ret); \ + break; \ + case 8: \ + __put_user_asm(x, d, addr, __pu_ret); \ + break; \ + default: \ + __pu_ret = __put_user_bad(); \ + break; \ + } \ + } else { \ + __pu_ret = -EFAULT; \ + } \ + __pu_ret; \ +}) + +#define __put_user_nocheck(x, addr, size) ({ \ + register int __pu_ret; \ + switch (size) { \ + case 1: __put_user_asm(x, b, addr, __pu_ret); break; \ + case 2: __put_user_asm(x, h, addr, __pu_ret); break; \ + case 4: __put_user_asm(x, , addr, __pu_ret); break; \ + case 8: __put_user_asm(x, d, addr, __pu_ret); break; \ + default: __pu_ret = __put_user_bad(); break; \ + } \ + __pu_ret; \ +}) + +#define __put_user_asm(x, size, addr, ret) \ __asm__ __volatile__( \ - "/* Put user asm, inline. */\n" \ -"1:\t" "st"#size " %1, %2\n\t" \ - "clr %0\n" \ -"2:\n\n\t" \ - ".section .fixup,#alloc,#execinstr\n\t" \ - ".align 4\n" \ -"3:\n\t" \ - "b 2b\n\t" \ - " mov %3, %0\n\t" \ - ".previous\n\n\t" \ - ".section __ex_table,#alloc\n\t" \ - ".align 4\n\t" \ - ".word 1b, 3b\n\t" \ - ".previous\n\n\t" \ - : "=&r" (ret) : "r" (x), "m" (*__m(addr)), \ - "i" (-EFAULT)) - -extern int __put_user_bad(void); - -#define __get_user_check(x,addr,size,type) ({ \ -register int __gu_ret; \ -register unsigned long __gu_val; \ -if (__access_ok(addr,size)) { \ -switch (size) { \ -case 1: __get_user_asm(__gu_val,ub,addr,__gu_ret); break; \ -case 2: __get_user_asm(__gu_val,uh,addr,__gu_ret); break; \ -case 4: __get_user_asm(__gu_val,,addr,__gu_ret); break; \ -case 8: __get_user_asm(__gu_val,d,addr,__gu_ret); break; \ -default: __gu_val = 0; __gu_ret = __get_user_bad(); break; \ -} } else { __gu_val = 0; __gu_ret = -EFAULT; } x = (type) __gu_val; __gu_ret; }) - -#define __get_user_check_ret(x,addr,size,type,retval) ({ \ -register unsigned long __gu_val __asm__ ("l1"); \ -if (__access_ok(addr,size)) { \ -switch (size) { \ -case 1: __get_user_asm_ret(__gu_val,ub,addr,retval); break; \ -case 2: __get_user_asm_ret(__gu_val,uh,addr,retval); break; \ -case 4: __get_user_asm_ret(__gu_val,,addr,retval); break; \ -case 8: __get_user_asm_ret(__gu_val,d,addr,retval); break; \ -default: if (__get_user_bad()) return retval; \ -} x = (type) __gu_val; } else return retval; }) - -#define __get_user_nocheck(x,addr,size,type) ({ \ -register int __gu_ret; \ -register unsigned long __gu_val; \ -switch (size) { \ -case 1: __get_user_asm(__gu_val,ub,addr,__gu_ret); break; \ -case 2: __get_user_asm(__gu_val,uh,addr,__gu_ret); break; \ -case 4: __get_user_asm(__gu_val,,addr,__gu_ret); break; \ -case 8: __get_user_asm(__gu_val,d,addr,__gu_ret); break; \ -default: __gu_val = 0; __gu_ret = __get_user_bad(); break; \ -} x = (type) __gu_val; __gu_ret; }) - -#define __get_user_nocheck_ret(x,addr,size,type,retval) ({ \ -register unsigned long __gu_val __asm__ ("l1"); \ -switch (size) { \ -case 1: __get_user_asm_ret(__gu_val,ub,addr,retval); break; \ -case 2: __get_user_asm_ret(__gu_val,uh,addr,retval); break; \ -case 4: __get_user_asm_ret(__gu_val,,addr,retval); break; \ -case 8: __get_user_asm_ret(__gu_val,d,addr,retval); break; \ -default: if (__get_user_bad()) return retval; \ -} x = (type) __gu_val; }) - -#define __get_user_asm(x,size,addr,ret) \ + "/* Put user asm, inline. */\n" \ + "1:\t" "st"#size " %1, %2\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\t" \ + ".previous\n\n\t" \ + ".section __ex_table,#alloc\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\t" \ + ".previous\n\n\t" \ + : "=&r" (ret) : "r" (x), "m" (*__m(addr)), \ + "i" (-EFAULT)) + +int __put_user_bad(void); + +#define __get_user_check(x, addr, size, type) ({ \ + register int __gu_ret; \ + register unsigned long __gu_val; \ + if (__access_ok(addr, size)) { \ + switch (size) { \ + case 1: \ + __get_user_asm(__gu_val, ub, addr, __gu_ret); \ + break; \ + case 2: \ + __get_user_asm(__gu_val, uh, addr, __gu_ret); \ + break; \ + case 4: \ + __get_user_asm(__gu_val, , addr, __gu_ret); \ + break; \ + case 8: \ + __get_user_asm(__gu_val, d, addr, __gu_ret); \ + break; \ + default: \ + __gu_val = 0; \ + __gu_ret = __get_user_bad(); \ + break; \ + } \ + } else { \ + __gu_val = 0; \ + __gu_ret = -EFAULT; \ + } \ + x = (__force type) __gu_val; \ + __gu_ret; \ +}) + +#define __get_user_nocheck(x, addr, size, type) ({ \ + register int __gu_ret; \ + register unsigned long __gu_val; \ + switch (size) { \ + case 1: __get_user_asm(__gu_val, ub, addr, __gu_ret); break; \ + case 2: __get_user_asm(__gu_val, uh, addr, __gu_ret); break; \ + case 4: __get_user_asm(__gu_val, , addr, __gu_ret); break; \ + case 8: __get_user_asm(__gu_val, d, addr, __gu_ret); break; \ + default: \ + __gu_val = 0; \ + __gu_ret = __get_user_bad(); \ + break; \ + } \ + x = (__force type) __gu_val; \ + __gu_ret; \ +}) + +#define __get_user_asm(x, size, addr, ret) \ __asm__ __volatile__( \ - "/* Get user asm, inline. */\n" \ -"1:\t" "ld"#size " %2, %1\n\t" \ - "clr %0\n" \ -"2:\n\n\t" \ - ".section .fixup,#alloc,#execinstr\n\t" \ - ".align 4\n" \ -"3:\n\t" \ - "clr %1\n\t" \ - "b 2b\n\t" \ - " mov %3, %0\n\n\t" \ - ".previous\n\t" \ - ".section __ex_table,#alloc\n\t" \ - ".align 4\n\t" \ - ".word 1b, 3b\n\n\t" \ - ".previous\n\t" \ - : "=&r" (ret), "=&r" (x) : "m" (*__m(addr)), \ - "i" (-EFAULT)) - -#define __get_user_asm_ret(x,size,addr,retval) \ -if (__builtin_constant_p(retval) && retval == -EFAULT) \ -__asm__ __volatile__( \ - "/* Get user asm ret, inline. */\n" \ -"1:\t" "ld"#size " %1, %0\n\n\t" \ - ".section __ex_table,#alloc\n\t" \ - ".align 4\n\t" \ - ".word 1b,__ret_efault\n\n\t" \ - ".previous\n\t" \ - : "=&r" (x) : "m" (*__m(addr))); \ -else \ -__asm__ __volatile__( \ - "/* Get user asm ret, inline. */\n" \ -"1:\t" "ld"#size " %1, %0\n\n\t" \ - ".section .fixup,#alloc,#execinstr\n\t" \ - ".align 4\n" \ -"3:\n\t" \ - "ret\n\t" \ - " restore %%g0, %2, %%o0\n\n\t" \ - ".previous\n\t" \ - ".section __ex_table,#alloc\n\t" \ - ".align 4\n\t" \ - ".word 1b, 3b\n\n\t" \ - ".previous\n\t" \ - : "=&r" (x) : "m" (*__m(addr)), "i" (retval)) - -extern int __get_user_bad(void); - -extern unsigned long __copy_user(void __user *to, const void __user *from, unsigned long size); - -static inline unsigned long copy_to_user(void __user *to, const void *from, unsigned long n) -{ - if (n && __access_ok((unsigned long) to, n)) - return __copy_user(to, (__force void __user *) from, n); - else - return n; -} - -static inline unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n) + "/* Get user asm, inline. */\n" \ + "1:\t" "ld"#size " %2, %1\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "clr %1\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\n\t" \ + ".previous\n\t" \ + ".section __ex_table,#alloc\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\n\t" \ + ".previous\n\t" \ + : "=&r" (ret), "=&r" (x) : "m" (*__m(addr)), \ + "i" (-EFAULT)) + +int __get_user_bad(void); + +unsigned long __copy_user(void __user *to, const void __user *from, unsigned long size); + +static inline unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n) { return __copy_user(to, (__force void __user *) from, n); } -static inline unsigned long copy_from_user(void *to, const void __user *from, unsigned long n) -{ - if (n && __access_ok((unsigned long) from, n)) - return __copy_user((__force void __user *) to, from, n); - else - return n; -} - -static inline unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n) +static inline unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n) { return __copy_user((__force void __user *) to, from, n); } -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER static inline unsigned long __clear_user(void __user *addr, unsigned long size) { unsigned long ret; __asm__ __volatile__ ( - ".section __ex_table,#alloc\n\t" - ".align 4\n\t" - ".word 1f,3\n\t" - ".previous\n\t" "mov %2, %%o1\n" - "1:\n\t" "call __bzero\n\t" " mov %1, %%o0\n\t" "mov %%o0, %0\n" @@ -300,15 +211,12 @@ static inline unsigned long __clear_user(void __user *addr, unsigned long size) static inline unsigned long clear_user(void __user *addr, unsigned long n) { - if (n && __access_ok((unsigned long) addr, n)) + if (n && __access_ok(addr, n)) return __clear_user(addr, n); else return n; } -extern __must_check long strlen_user(const char __user *str); -extern __must_check long strnlen_user(const char __user *str, long n); - -#endif /* __ASSEMBLY__ */ +__must_check long strnlen_user(const char __user *str, long n); #endif /* _ASM_UACCESS_H */ diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index e562d3caee57..b825a5dd0210 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_UACCESS_H #define _ASM_UACCESS_H @@ -5,79 +6,46 @@ * User space memory access functions */ -#ifdef __KERNEL__ -#include <linux/errno.h> #include <linux/compiler.h> #include <linux/string.h> -#include <linux/thread_info.h> +#include <linux/mm_types.h> #include <asm/asi.h> #include <asm/spitfire.h> -#include <asm-generic/uaccess-unaligned.h> -#endif - -#ifndef __ASSEMBLY__ +#include <asm/pgtable.h> #include <asm/processor.h> +#include <asm-generic/access_ok.h> /* * Sparc64 is segmented, though more like the M68K than the I386. * We use the secondary ASI to address user memory, which references a * completely different VM map, thus there is zero chance of the user * doing something queer and tricking us into poking kernel memory. - * - * What is left here is basically what is needed for the other parts of - * the kernel that expect to be able to manipulate, erum, "segments". - * Or perhaps more properly, permissions. - * - * "For historical reasons, these macros are grossly misnamed." -Linus */ -#define KERNEL_DS ((mm_segment_t) { ASI_P }) -#define USER_DS ((mm_segment_t) { ASI_AIUS }) /* har har har */ - -#define VERIFY_READ 0 -#define VERIFY_WRITE 1 - -#define get_fs() ((mm_segment_t){(current_thread_info()->current_ds)}) -#define get_ds() (KERNEL_DS) - -#define segment_eq(a,b) ((a).seg == (b).seg) - -#define set_fs(val) \ -do { \ - current_thread_info()->current_ds =(val).seg; \ - __asm__ __volatile__ ("wr %%g0, %0, %%asi" : : "r" ((val).seg)); \ -} while(0) - -static inline int __access_ok(const void __user * addr, unsigned long size) +/* + * Test whether a block of memory is a valid user space address. + * Returns 0 if the range is valid, nonzero otherwise. + */ +static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit) { - return 1; -} + if (__builtin_constant_p(size)) + return addr > limit - size; -static inline int access_ok(int type, const void __user * addr, unsigned long size) -{ - return 1; -} + addr += size; + if (addr < size) + return true; -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ + return addr > limit; +} -struct exception_table_entry { - unsigned int insn, fixup; -}; +#define __range_not_ok(addr, size, limit) \ +({ \ + __chk_user_ptr(addr); \ + __chk_range_not_ok((unsigned long __force)(addr), size, limit); \ +}) -extern void __ret_efault(void); -extern void __retl_efault(void); +void __retl_efault(void); /* Uh, these should become the main single-value transfer routines.. * They automatically use the right size if we just have the right @@ -88,188 +56,197 @@ extern void __retl_efault(void); * of a performance impact. Thus we have a few rather ugly macros here, * and hide all the ugliness from the user. */ -#define put_user(x,ptr) ({ \ -unsigned long __pu_addr = (unsigned long)(ptr); \ -__chk_user_ptr(ptr); \ -__put_user_nocheck((__typeof__(*(ptr)))(x),__pu_addr,sizeof(*(ptr))); }) +#define put_user(x, ptr) ({ \ + unsigned long __pu_addr = (unsigned long)(ptr); \ + __chk_user_ptr(ptr); \ + __put_user_nocheck((__typeof__(*(ptr)))(x), __pu_addr, sizeof(*(ptr)));\ +}) -#define get_user(x,ptr) ({ \ -unsigned long __gu_addr = (unsigned long)(ptr); \ -__chk_user_ptr(ptr); \ -__get_user_nocheck((x),__gu_addr,sizeof(*(ptr)),__typeof__(*(ptr))); }) +#define get_user(x, ptr) ({ \ + unsigned long __gu_addr = (unsigned long)(ptr); \ + __chk_user_ptr(ptr); \ + __get_user_nocheck((x), __gu_addr, sizeof(*(ptr)), __typeof__(*(ptr)));\ +}) -#define __put_user(x,ptr) put_user(x,ptr) -#define __get_user(x,ptr) get_user(x,ptr) +#define __put_user(x, ptr) put_user(x, ptr) +#define __get_user(x, ptr) get_user(x, ptr) struct __large_struct { unsigned long buf[100]; }; #define __m(x) ((struct __large_struct *)(x)) -#define __put_user_nocheck(data,addr,size) ({ \ -register int __pu_ret; \ -switch (size) { \ -case 1: __put_user_asm(data,b,addr,__pu_ret); break; \ -case 2: __put_user_asm(data,h,addr,__pu_ret); break; \ -case 4: __put_user_asm(data,w,addr,__pu_ret); break; \ -case 8: __put_user_asm(data,x,addr,__pu_ret); break; \ -default: __pu_ret = __put_user_bad(); break; \ -} __pu_ret; }) - -#define __put_user_asm(x,size,addr,ret) \ +#define __put_kernel_nofault(dst, src, type, label) \ +do { \ + type *addr = (type __force *)(dst); \ + type data = *(type *)src; \ + register int __pu_ret; \ + switch (sizeof(type)) { \ + case 1: __put_kernel_asm(data, b, addr, __pu_ret); break; \ + case 2: __put_kernel_asm(data, h, addr, __pu_ret); break; \ + case 4: __put_kernel_asm(data, w, addr, __pu_ret); break; \ + case 8: __put_kernel_asm(data, x, addr, __pu_ret); break; \ + default: __pu_ret = __put_user_bad(); break; \ + } \ + if (__pu_ret) \ + goto label; \ +} while (0) + +#define __put_kernel_asm(x, size, addr, ret) \ __asm__ __volatile__( \ - "/* Put user asm, inline. */\n" \ -"1:\t" "st"#size "a %1, [%2] %%asi\n\t" \ - "clr %0\n" \ -"2:\n\n\t" \ - ".section .fixup,#alloc,#execinstr\n\t" \ - ".align 4\n" \ -"3:\n\t" \ - "sethi %%hi(2b), %0\n\t" \ - "jmpl %0 + %%lo(2b), %%g0\n\t" \ - " mov %3, %0\n\n\t" \ - ".previous\n\t" \ - ".section __ex_table,\"a\"\n\t" \ - ".align 4\n\t" \ - ".word 1b, 3b\n\t" \ - ".previous\n\n\t" \ - : "=r" (ret) : "r" (x), "r" (__m(addr)), \ - "i" (-EFAULT)) - -extern int __put_user_bad(void); - -#define __get_user_nocheck(data,addr,size,type) ({ \ -register int __gu_ret; \ -register unsigned long __gu_val; \ -switch (size) { \ -case 1: __get_user_asm(__gu_val,ub,addr,__gu_ret); break; \ -case 2: __get_user_asm(__gu_val,uh,addr,__gu_ret); break; \ -case 4: __get_user_asm(__gu_val,uw,addr,__gu_ret); break; \ -case 8: __get_user_asm(__gu_val,x,addr,__gu_ret); break; \ -default: __gu_val = 0; __gu_ret = __get_user_bad(); break; \ -} data = (type) __gu_val; __gu_ret; }) - -#define __get_user_nocheck_ret(data,addr,size,type,retval) ({ \ -register unsigned long __gu_val __asm__ ("l1"); \ -switch (size) { \ -case 1: __get_user_asm_ret(__gu_val,ub,addr,retval); break; \ -case 2: __get_user_asm_ret(__gu_val,uh,addr,retval); break; \ -case 4: __get_user_asm_ret(__gu_val,uw,addr,retval); break; \ -case 8: __get_user_asm_ret(__gu_val,x,addr,retval); break; \ -default: if (__get_user_bad()) return retval; \ -} data = (type) __gu_val; }) - -#define __get_user_asm(x,size,addr,ret) \ + "/* Put kernel asm, inline. */\n" \ + "1:\t" "st"#size " %1, [%2]\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\n\t" \ + ".previous\n\t" \ + ".section __ex_table,\"a\"\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\t" \ + ".previous\n\n\t" \ + : "=r" (ret) : "r" (x), "r" (__m(addr)), \ + "i" (-EFAULT)) + +#define __put_user_nocheck(data, addr, size) ({ \ + register int __pu_ret; \ + switch (size) { \ + case 1: __put_user_asm(data, b, addr, __pu_ret); break; \ + case 2: __put_user_asm(data, h, addr, __pu_ret); break; \ + case 4: __put_user_asm(data, w, addr, __pu_ret); break; \ + case 8: __put_user_asm(data, x, addr, __pu_ret); break; \ + default: __pu_ret = __put_user_bad(); break; \ + } \ + __pu_ret; \ +}) + +#define __put_user_asm(x, size, addr, ret) \ __asm__ __volatile__( \ - "/* Get user asm, inline. */\n" \ -"1:\t" "ld"#size "a [%2] %%asi, %1\n\t" \ - "clr %0\n" \ -"2:\n\n\t" \ - ".section .fixup,#alloc,#execinstr\n\t" \ - ".align 4\n" \ -"3:\n\t" \ - "sethi %%hi(2b), %0\n\t" \ - "clr %1\n\t" \ - "jmpl %0 + %%lo(2b), %%g0\n\t" \ - " mov %3, %0\n\n\t" \ - ".previous\n\t" \ - ".section __ex_table,\"a\"\n\t" \ - ".align 4\n\t" \ - ".word 1b, 3b\n\n\t" \ - ".previous\n\t" \ - : "=r" (ret), "=r" (x) : "r" (__m(addr)), \ - "i" (-EFAULT)) - -#define __get_user_asm_ret(x,size,addr,retval) \ -if (__builtin_constant_p(retval) && retval == -EFAULT) \ + "/* Put user asm, inline. */\n" \ + "1:\t" "st"#size "a %1, [%2] %%asi\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\n\t" \ + ".previous\n\t" \ + ".section __ex_table,\"a\"\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\t" \ + ".previous\n\n\t" \ + : "=r" (ret) : "r" (x), "r" (__m(addr)), \ + "i" (-EFAULT)) + +int __put_user_bad(void); + +#define __get_kernel_nofault(dst, src, type, label) \ +do { \ + type *addr = (type __force *)(src); \ + register int __gu_ret; \ + register unsigned long __gu_val; \ + switch (sizeof(type)) { \ + case 1: __get_kernel_asm(__gu_val, ub, addr, __gu_ret); break; \ + case 2: __get_kernel_asm(__gu_val, uh, addr, __gu_ret); break; \ + case 4: __get_kernel_asm(__gu_val, uw, addr, __gu_ret); break; \ + case 8: __get_kernel_asm(__gu_val, x, addr, __gu_ret); break; \ + default: \ + __gu_val = 0; \ + __gu_ret = __get_user_bad(); \ + break; \ + } \ + if (__gu_ret) \ + goto label; \ + *(type *)dst = (__force type) __gu_val; \ +} while (0) +#define __get_kernel_asm(x, size, addr, ret) \ __asm__ __volatile__( \ - "/* Get user asm ret, inline. */\n" \ -"1:\t" "ld"#size "a [%1] %%asi, %0\n\n\t" \ - ".section __ex_table,\"a\"\n\t" \ - ".align 4\n\t" \ - ".word 1b,__ret_efault\n\n\t" \ - ".previous\n\t" \ - : "=r" (x) : "r" (__m(addr))); \ -else \ + "/* Get kernel asm, inline. */\n" \ + "1:\t" "ld"#size " [%2], %1\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "clr %1\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\n\t" \ + ".previous\n\t" \ + ".section __ex_table,\"a\"\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\n\t" \ + ".previous\n\t" \ + : "=r" (ret), "=r" (x) : "r" (__m(addr)), \ + "i" (-EFAULT)) + +#define __get_user_nocheck(data, addr, size, type) ({ \ + register int __gu_ret; \ + register unsigned long __gu_val; \ + switch (size) { \ + case 1: __get_user_asm(__gu_val, ub, addr, __gu_ret); break; \ + case 2: __get_user_asm(__gu_val, uh, addr, __gu_ret); break; \ + case 4: __get_user_asm(__gu_val, uw, addr, __gu_ret); break; \ + case 8: __get_user_asm(__gu_val, x, addr, __gu_ret); break; \ + default: \ + __gu_val = 0; \ + __gu_ret = __get_user_bad(); \ + break; \ + } \ + data = (__force type) __gu_val; \ + __gu_ret; \ +}) + +#define __get_user_asm(x, size, addr, ret) \ __asm__ __volatile__( \ - "/* Get user asm ret, inline. */\n" \ -"1:\t" "ld"#size "a [%1] %%asi, %0\n\n\t" \ - ".section .fixup,#alloc,#execinstr\n\t" \ - ".align 4\n" \ -"3:\n\t" \ - "ret\n\t" \ - " restore %%g0, %2, %%o0\n\n\t" \ - ".previous\n\t" \ - ".section __ex_table,\"a\"\n\t" \ - ".align 4\n\t" \ - ".word 1b, 3b\n\n\t" \ - ".previous\n\t" \ - : "=r" (x) : "r" (__m(addr)), "i" (retval)) - -extern int __get_user_bad(void); - -extern unsigned long __must_check ___copy_from_user(void *to, - const void __user *from, - unsigned long size); -extern unsigned long copy_from_user_fixup(void *to, const void __user *from, - unsigned long size); -static inline unsigned long __must_check -copy_from_user(void *to, const void __user *from, unsigned long size) -{ - unsigned long ret = ___copy_from_user(to, from, size); - - if (unlikely(ret)) - ret = copy_from_user_fixup(to, from, size); - - return ret; -} -#define __copy_from_user copy_from_user - -extern unsigned long __must_check ___copy_to_user(void __user *to, - const void *from, - unsigned long size); -extern unsigned long copy_to_user_fixup(void __user *to, const void *from, - unsigned long size); -static inline unsigned long __must_check -copy_to_user(void __user *to, const void *from, unsigned long size) -{ - unsigned long ret = ___copy_to_user(to, from, size); - - if (unlikely(ret)) - ret = copy_to_user_fixup(to, from, size); - return ret; -} -#define __copy_to_user copy_to_user - -extern unsigned long __must_check ___copy_in_user(void __user *to, - const void __user *from, - unsigned long size); -extern unsigned long copy_in_user_fixup(void __user *to, void __user *from, - unsigned long size); -static inline unsigned long __must_check -copy_in_user(void __user *to, void __user *from, unsigned long size) -{ - unsigned long ret = ___copy_in_user(to, from, size); - - if (unlikely(ret)) - ret = copy_in_user_fixup(to, from, size); - return ret; -} -#define __copy_in_user copy_in_user - -extern unsigned long __must_check __clear_user(void __user *, unsigned long); + "/* Get user asm, inline. */\n" \ + "1:\t" "ld"#size "a [%2] %%asi, %1\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "clr %1\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\n\t" \ + ".previous\n\t" \ + ".section __ex_table,\"a\"\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\n\t" \ + ".previous\n\t" \ + : "=r" (ret), "=r" (x) : "r" (__m(addr)), \ + "i" (-EFAULT)) + +int __get_user_bad(void); + +unsigned long __must_check raw_copy_from_user(void *to, + const void __user *from, + unsigned long size); + +unsigned long __must_check raw_copy_to_user(void __user *to, + const void *from, + unsigned long size); +#define INLINE_COPY_FROM_USER +#define INLINE_COPY_TO_USER + +unsigned long __must_check raw_copy_in_user(void __user *to, + const void __user *from, + unsigned long size); + +unsigned long __must_check __clear_user(void __user *, unsigned long); #define clear_user __clear_user -extern __must_check long strlen_user(const char __user *str); -extern __must_check long strnlen_user(const char __user *str, long n); - -#define __copy_to_user_inatomic ___copy_to_user -#define __copy_from_user_inatomic ___copy_from_user +__must_check long strnlen_user(const char __user *str, long n); struct pt_regs; -extern unsigned long compute_effective_address(struct pt_regs *, - unsigned int insn, - unsigned int rd); - -#endif /* __ASSEMBLY__ */ +unsigned long compute_effective_address(struct pt_regs *, + unsigned int insn, + unsigned int rd); #endif /* _ASM_UACCESS_H */ diff --git a/arch/sparc/include/asm/unaligned.h b/arch/sparc/include/asm/unaligned.h deleted file mode 100644 index 11d2d5fb5902..000000000000 --- a/arch/sparc/include/asm/unaligned.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef _ASM_SPARC_UNALIGNED_H -#define _ASM_SPARC_UNALIGNED_H - -#include <linux/unaligned/be_struct.h> -#include <linux/unaligned/le_byteshift.h> -#include <linux/unaligned/generic.h> -#define get_unaligned __get_unaligned_be -#define put_unaligned __put_unaligned_be - -#endif /* _ASM_SPARC_UNALIGNED_H */ diff --git a/arch/sparc/include/asm/unistd.h b/arch/sparc/include/asm/unistd.h index dfa53fdd5cbc..3380411a4537 100644 --- a/arch/sparc/include/asm/unistd.h +++ b/arch/sparc/include/asm/unistd.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * System calls under the Sparc. * @@ -16,24 +17,25 @@ #include <uapi/asm/unistd.h> +#define NR_syscalls __NR_syscalls + #ifdef __32bit_syscall_numbers__ #else #define __NR_time 231 /* Linux sparc32 */ #endif +#define __ARCH_WANT_NEW_STAT #define __ARCH_WANT_OLD_READDIR #define __ARCH_WANT_STAT64 #define __ARCH_WANT_SYS_ALARM #define __ARCH_WANT_SYS_GETHOSTNAME #define __ARCH_WANT_SYS_PAUSE -#define __ARCH_WANT_SYS_SGETMASK #define __ARCH_WANT_SYS_SIGNAL -#define __ARCH_WANT_SYS_TIME -#define __ARCH_WANT_SYS_UTIME +#define __ARCH_WANT_SYS_TIME32 +#define __ARCH_WANT_SYS_UTIME32 #define __ARCH_WANT_SYS_WAITPID #define __ARCH_WANT_SYS_SOCKETCALL #define __ARCH_WANT_SYS_FADVISE64 #define __ARCH_WANT_SYS_GETPGRP -#define __ARCH_WANT_SYS_LLSEEK #define __ARCH_WANT_SYS_NICE #define __ARCH_WANT_SYS_OLDUMOUNT #define __ARCH_WANT_SYS_SIGPENDING @@ -41,8 +43,23 @@ #ifdef __32bit_syscall_numbers__ #define __ARCH_WANT_SYS_IPC #else -#define __ARCH_WANT_COMPAT_SYS_TIME +#define __ARCH_WANT_SYS_TIME +#define __ARCH_WANT_SYS_UTIME #define __ARCH_WANT_COMPAT_SYS_SENDFILE +#define __ARCH_WANT_COMPAT_STAT +#endif + +#define __ARCH_BROKEN_SYS_CLONE3 + +#ifdef __32bit_syscall_numbers__ +/* Sparc 32-bit only has the "setresuid32", "getresuid32" variants, + * it never had the plain ones and there is no value to adding those + * old versions into the syscall table. + */ +#define __IGNORE_setresuid +#define __IGNORE_getresuid +#define __IGNORE_setresgid +#define __IGNORE_getresgid #endif #endif /* _SPARC_UNISTD_H */ diff --git a/arch/sparc/include/asm/upa.h b/arch/sparc/include/asm/upa.h index 5b1633223f92..b1df3a7f40ed 100644 --- a/arch/sparc/include/asm/upa.h +++ b/arch/sparc/include/asm/upa.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_UPA_H #define _SPARC64_UPA_H @@ -23,7 +24,7 @@ #define UPA_PORTID_ID 0x000000000000ffff /* Module Identification bits */ /* UPA I/O space accessors */ -#if defined(__KERNEL__) && !defined(__ASSEMBLY__) +#if defined(__KERNEL__) && !defined(__ASSEMBLER__) static inline unsigned char _upa_readb(unsigned long addr) { unsigned char ret; @@ -104,6 +105,6 @@ static inline void _upa_writeq(unsigned long q, unsigned long addr) #define upa_writew(__w, __addr) (_upa_writew((__w), (unsigned long)(__addr))) #define upa_writel(__l, __addr) (_upa_writel((__l), (unsigned long)(__addr))) #define upa_writeq(__q, __addr) (_upa_writeq((__q), (unsigned long)(__addr))) -#endif /* __KERNEL__ && !__ASSEMBLY__ */ +#endif /* __KERNEL__ && !__ASSEMBLER__ */ #endif /* !(_SPARC64_UPA_H) */ diff --git a/arch/sparc/include/asm/uprobes.h b/arch/sparc/include/asm/uprobes.h new file mode 100644 index 000000000000..79938fb94016 --- /dev/null +++ b/arch/sparc/include/asm/uprobes.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_UPROBES_H +#define _ASM_UPROBES_H +/* + * User-space Probes (UProbes) for sparc + * + * Copyright (C) 2013 Oracle, Inc. + * + * Authors: + * Jose E. Marchesi <jose.marchesi@oracle.com> + * Eric Saint Etienne <eric.saint.etienne@oracle.com> + */ + +typedef u32 uprobe_opcode_t; + +#define MAX_UINSN_BYTES 4 +#define UPROBE_XOL_SLOT_BYTES (MAX_UINSN_BYTES * 2) + +#define UPROBE_SWBP_INSN_SIZE 4 +#define UPROBE_SWBP_INSN 0x91d02073 /* ta 0x73 */ +#define UPROBE_STP_INSN 0x91d02074 /* ta 0x74 */ + +#define ANNUL_BIT (1 << 29) + +struct arch_uprobe { + union { + u8 insn[MAX_UINSN_BYTES]; + u32 ixol; + }; +}; + +struct arch_uprobe_task { + u64 saved_tpc; + u64 saved_tnpc; +}; + +struct task_struct; +struct notifier_block; + +extern int arch_uprobe_analyze_insn(struct arch_uprobe *aup, struct mm_struct *mm, unsigned long addr); +extern int arch_uprobe_pre_xol(struct arch_uprobe *aup, struct pt_regs *regs); +extern int arch_uprobe_post_xol(struct arch_uprobe *aup, struct pt_regs *regs); +extern bool arch_uprobe_xol_was_trapped(struct task_struct *tsk); +extern int arch_uprobe_exception_notify(struct notifier_block *self, unsigned long val, void *data); +extern void arch_uprobe_abort_xol(struct arch_uprobe *aup, struct pt_regs *regs); + +#endif /* _ASM_UPROBES_H */ diff --git a/arch/sparc/include/asm/vaddrs.h b/arch/sparc/include/asm/vaddrs.h index c3dbcf902034..da567600c897 100644 --- a/arch/sparc/include/asm/vaddrs.h +++ b/arch/sparc/include/asm/vaddrs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC_VADDRS_H #define _SPARC_VADDRS_H @@ -30,14 +31,14 @@ */ #define SRMMU_NOCACHE_ALCRATIO 64 /* 256 pages per 64MB of system RAM */ -#ifndef __ASSEMBLY__ -#include <asm/kmap_types.h> +#ifndef __ASSEMBLER__ +#include <asm/kmap_size.h> enum fixed_addresses { FIX_HOLE, #ifdef CONFIG_HIGHMEM FIX_KMAP_BEGIN, - FIX_KMAP_END = (KM_TYPE_NR * NR_CPUS), + FIX_KMAP_END = (KM_MAX_IDX * NR_CPUS), #endif __end_of_fixed_addresses }; diff --git a/arch/sparc/include/asm/vdso.h b/arch/sparc/include/asm/vdso.h new file mode 100644 index 000000000000..59e79d35cd73 --- /dev/null +++ b/arch/sparc/include/asm/vdso.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#ifndef _ASM_SPARC_VDSO_H +#define _ASM_SPARC_VDSO_H + +struct vdso_image { + void *data; + unsigned long size; /* Always a multiple of PAGE_SIZE */ + + long sym_vvar_start; /* Negative offset to the vvar area */ +}; + +#ifdef CONFIG_SPARC64 +extern const struct vdso_image vdso_image_64_builtin; +#endif +#ifdef CONFIG_COMPAT +extern const struct vdso_image vdso_image_32_builtin; +#endif + +#endif /* _ASM_SPARC_VDSO_H */ diff --git a/arch/sparc/include/asm/vga.h b/arch/sparc/include/asm/vga.h deleted file mode 100644 index ec0e9967d93d..000000000000 --- a/arch/sparc/include/asm/vga.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Access to VGA videoram - * - * (c) 1998 Martin Mares <mj@ucw.cz> - */ - -#ifndef _LINUX_ASM_VGA_H_ -#define _LINUX_ASM_VGA_H_ - -#include <linux/bug.h> -#include <asm/types.h> - -#define VT_BUF_HAVE_RW - -#undef scr_writew -#undef scr_readw - -static inline void scr_writew(u16 val, u16 *addr) -{ - BUG_ON((long) addr >= 0); - - *addr = val; -} - -static inline u16 scr_readw(const u16 *addr) -{ - BUG_ON((long) addr >= 0); - - return *addr; -} - -#define VGA_MAP_MEM(x,s) (x) - -#endif diff --git a/arch/sparc/include/asm/video.h b/arch/sparc/include/asm/video.h new file mode 100644 index 000000000000..773717b6d491 --- /dev/null +++ b/arch/sparc/include/asm/video.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_VIDEO_H_ +#define _SPARC_VIDEO_H_ + +#include <linux/io.h> +#include <linux/types.h> + +#include <asm/page.h> + +struct device; + +#ifdef CONFIG_SPARC32 +static inline pgprot_t pgprot_framebuffer(pgprot_t prot, + unsigned long vm_start, unsigned long vm_end, + unsigned long offset) +{ + return prot; +} +#define pgprot_framebuffer pgprot_framebuffer +#endif + +#ifdef CONFIG_VIDEO +bool video_is_primary_device(struct device *dev); +#define video_is_primary_device video_is_primary_device +#endif + +static inline void fb_memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) +{ + sbus_memcpy_fromio(to, from, n); +} +#define fb_memcpy_fromio fb_memcpy_fromio + +static inline void fb_memcpy_toio(volatile void __iomem *to, const void *from, size_t n) +{ + sbus_memcpy_toio(to, from, n); +} +#define fb_memcpy_toio fb_memcpy_toio + +static inline void fb_memset_io(volatile void __iomem *addr, int c, size_t n) +{ + sbus_memset_io(addr, c, n); +} +#define fb_memset fb_memset_io + +#include <asm-generic/video.h> + +#endif /* _SPARC_VIDEO_H_ */ diff --git a/arch/sparc/include/asm/viking.h b/arch/sparc/include/asm/viking.h index 989930aeb093..bbb714de43c4 100644 --- a/arch/sparc/include/asm/viking.h +++ b/arch/sparc/include/asm/viking.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * viking.h: Defines specific to the GNU/Viking MBUS module. * This is SRMMU stuff. @@ -9,6 +10,7 @@ #include <asm/asi.h> #include <asm/mxcc.h> +#include <asm/pgtable.h> #include <asm/pgtsrmmu.h> /* Bits in the SRMMU control register for GNU/Viking modules. @@ -108,7 +110,7 @@ #define VIKING_PTAG_DIRTY 0x00010000 /* Block has been modified */ #define VIKING_PTAG_SHARED 0x00000100 /* Shared with some other cache */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ static inline void viking_flush_icache(void) { @@ -226,7 +228,7 @@ static inline unsigned long viking_hwprobe(unsigned long vaddr) : "=r" (val) : "r" (vaddr | 0x200), "i" (ASI_M_FLUSH_PROBE)); if ((val & SRMMU_ET_MASK) == SRMMU_ET_PTE) { - vaddr &= ~SRMMU_PGDIR_MASK; + vaddr &= ~PGDIR_MASK; vaddr >>= PAGE_SHIFT; return val | (vaddr << 8); } @@ -236,7 +238,7 @@ static inline unsigned long viking_hwprobe(unsigned long vaddr) : "=r" (val) : "r" (vaddr | 0x100), "i" (ASI_M_FLUSH_PROBE)); if ((val & SRMMU_ET_MASK) == SRMMU_ET_PTE) { - vaddr &= ~SRMMU_REAL_PMD_MASK; + vaddr &= ~PMD_MASK; vaddr >>= PAGE_SHIFT; return val | (vaddr << 8); } @@ -248,6 +250,6 @@ static inline unsigned long viking_hwprobe(unsigned long vaddr) return val; } -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* !(_SPARC_VIKING_H) */ diff --git a/arch/sparc/include/asm/vio.h b/arch/sparc/include/asm/vio.h index 432afa838861..0ca8c3463166 100644 --- a/arch/sparc/include/asm/vio.h +++ b/arch/sparc/include/asm/vio.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_VIO_H #define _SPARC64_VIO_H @@ -52,6 +53,7 @@ struct vio_ver_info { #define VDEV_NETWORK_SWITCH 0x02 #define VDEV_DISK 0x03 #define VDEV_DISK_SERVER 0x04 +#define VDEV_CONSOLE_CON 0x05 u8 resv1[3]; u64 resv2[5]; @@ -65,9 +67,10 @@ struct vio_dring_register { u16 options; #define VIO_TX_DRING 0x0001 #define VIO_RX_DRING 0x0002 +#define VIO_RX_DRING_DATA 0x0004 u16 resv; u32 num_cookies; - struct ldc_trans_cookie cookies[0]; + struct ldc_trans_cookie cookies[]; }; struct vio_dring_unregister { @@ -80,6 +83,8 @@ struct vio_dring_unregister { #define VIO_PKT_MODE 0x01 /* Packet based transfer */ #define VIO_DESC_MODE 0x02 /* In-band descriptors */ #define VIO_DRING_MODE 0x03 /* Descriptor rings */ +/* in vers >= 1.2, VIO_DRING_MODE is 0x04 and transfer mode is a bitmask */ +#define VIO_NEW_DRING_MODE 0x04 struct vio_dring_data { struct vio_msg_tag tag; @@ -118,12 +123,18 @@ struct vio_disk_attr_info { u8 vdisk_type; #define VD_DISK_TYPE_SLICE 0x01 /* Slice in block device */ #define VD_DISK_TYPE_DISK 0x02 /* Entire block device */ - u16 resv1; + u8 vdisk_mtype; /* v1.1 */ +#define VD_MEDIA_TYPE_FIXED 0x01 /* Fixed device */ +#define VD_MEDIA_TYPE_CD 0x02 /* CD Device */ +#define VD_MEDIA_TYPE_DVD 0x03 /* DVD Device */ + u8 resv1; u32 vdisk_block_size; u64 operations; - u64 vdisk_size; + u64 vdisk_size; /* v1.1 */ u64 max_xfer_size; - u64 resv2[2]; + u32 phys_block_size; /* v1.2 */ + u32 resv2; + u64 resv3[1]; }; struct vio_disk_desc { @@ -150,7 +161,7 @@ struct vio_disk_desc { u64 size; u32 ncookies; u32 resv2; - struct ldc_trans_cookie cookies[0]; + struct ldc_trans_cookie cookies[]; }; #define VIO_DISK_VNAME_LEN 8 @@ -189,13 +200,13 @@ struct vio_disk_devid { u16 resv; u16 type; u32 len; - char id[0]; + char id[]; }; struct vio_disk_efi { u64 lba; u64 len; - char data[0]; + char data[]; }; /* VIO net specific structures and defines */ @@ -205,10 +216,20 @@ struct vio_net_attr_info { u8 addr_type; #define VNET_ADDR_ETHERMAC 0x01 u16 ack_freq; - u32 resv1; + u8 plnk_updt; +#define PHYSLINK_UPDATE_NONE 0x00 +#define PHYSLINK_UPDATE_STATE 0x01 +#define PHYSLINK_UPDATE_STATE_ACK 0x02 +#define PHYSLINK_UPDATE_STATE_NACK 0x03 + u8 options; + u16 resv1; u64 addr; u64 mtu; - u64 resv2[3]; + u16 cflags; +#define VNET_LSO_IPV4_CAPAB 0x0001 + u16 ipv4_lso_maxlen; + u32 resv2; + u64 resv3[2]; }; #define VNET_NUM_MCAST 7 @@ -225,9 +246,28 @@ struct vio_net_desc { struct vio_dring_hdr hdr; u32 size; u32 ncookies; - struct ldc_trans_cookie cookies[0]; + struct ldc_trans_cookie cookies[]; +}; + +struct vio_net_dext { + u8 flags; +#define VNET_PKT_HASH 0x01 +#define VNET_PKT_HCK_IPV4_HDRCKSUM 0x02 +#define VNET_PKT_HCK_FULLCKSUM 0x04 +#define VNET_PKT_IPV4_LSO 0x08 +#define VNET_PKT_HCK_IPV4_HDRCKSUM_OK 0x10 +#define VNET_PKT_HCK_FULLCKSUM_OK 0x20 + + u8 vnet_hashval; + u16 ipv4_lso_mss; + u32 resv3; }; +static inline struct vio_net_dext *vio_net_ext(struct vio_net_desc *desc) +{ + return (struct vio_net_dext *)&desc->cookies[2]; +} + #define VIO_MAX_RING_COOKIES 24 struct vio_dring_state { @@ -244,6 +284,14 @@ struct vio_dring_state { struct ldc_trans_cookie cookies[VIO_MAX_RING_COOKIES]; }; +#define VIO_TAG_SIZE (sizeof(struct vio_msg_tag)) +#define VIO_VCC_MTU_SIZE (LDC_PACKET_SIZE - VIO_TAG_SIZE) + +struct vio_vcc { + struct vio_msg_tag tag; + char data[VIO_VCC_MTU_SIZE]; +}; + static inline void *vio_dring_cur(struct vio_dring_state *dr) { return dr->base + (dr->entry_size * dr->prod); @@ -259,26 +307,52 @@ static inline u32 vio_dring_avail(struct vio_dring_state *dr, unsigned int ring_size) { return (dr->pending - - ((dr->prod - dr->cons) & (ring_size - 1))); + ((dr->prod - dr->cons) & (ring_size - 1)) - 1); +} + +static inline u32 vio_dring_next(struct vio_dring_state *dr, u32 index) +{ + if (++index == dr->num_entries) + index = 0; + return index; +} + +static inline u32 vio_dring_prev(struct vio_dring_state *dr, u32 index) +{ + if (index == 0) + return dr->num_entries - 1; + else + return index - 1; } #define VIO_MAX_TYPE_LEN 32 +#define VIO_MAX_NAME_LEN 32 #define VIO_MAX_COMPAT_LEN 64 struct vio_dev { u64 mp; struct device_node *dp; + char node_name[VIO_MAX_NAME_LEN]; char type[VIO_MAX_TYPE_LEN]; char compat[VIO_MAX_COMPAT_LEN]; int compat_len; u64 dev_no; + unsigned long port_id; unsigned long channel_id; unsigned int tx_irq; unsigned int rx_irq; + u64 rx_ino; + u64 tx_ino; + + /* Handle to the root of "channel-devices" sub-tree in MDESC */ + u64 cdev_handle; + + /* MD specific data used to identify the vdev in MD */ + union md_node_info md_node_info; struct device dev; }; @@ -288,10 +362,11 @@ struct vio_driver { struct list_head node; const struct vio_device_id *id_table; int (*probe)(struct vio_dev *dev, const struct vio_device_id *id); - int (*remove)(struct vio_dev *dev); + void (*remove)(struct vio_dev *dev); void (*shutdown)(struct vio_dev *dev); unsigned long driver_data; struct device_driver driver; + bool no_irq; }; struct vio_version { @@ -366,46 +441,68 @@ struct vio_driver_state { struct vio_driver_ops *ops; }; +static inline bool vio_version_before(struct vio_driver_state *vio, + u16 major, u16 minor) +{ + u32 have = (u32)vio->ver.major << 16 | vio->ver.minor; + u32 want = (u32)major << 16 | minor; + + return have < want; +} + +static inline bool vio_version_after(struct vio_driver_state *vio, + u16 major, u16 minor) +{ + u32 have = (u32)vio->ver.major << 16 | vio->ver.minor; + u32 want = (u32)major << 16 | minor; + + return have > want; +} + +static inline bool vio_version_after_eq(struct vio_driver_state *vio, + u16 major, u16 minor) +{ + u32 have = (u32)vio->ver.major << 16 | vio->ver.minor; + u32 want = (u32)major << 16 | minor; + + return have >= want; +} + #define viodbg(TYPE, f, a...) \ do { if (vio->debug & VIO_DEBUG_##TYPE) \ printk(KERN_INFO "vio: ID[%lu] " f, \ vio->vdev->channel_id, ## a); \ } while (0) -extern int __vio_register_driver(struct vio_driver *drv, struct module *owner, +int __vio_register_driver(struct vio_driver *drv, struct module *owner, const char *mod_name); /* * vio_register_driver must be a macro so that KBUILD_MODNAME can be expanded */ #define vio_register_driver(driver) \ __vio_register_driver(driver, THIS_MODULE, KBUILD_MODNAME) -extern void vio_unregister_driver(struct vio_driver *drv); - -static inline struct vio_driver *to_vio_driver(struct device_driver *drv) -{ - return container_of(drv, struct vio_driver, driver); -} - -static inline struct vio_dev *to_vio_dev(struct device *dev) -{ - return container_of(dev, struct vio_dev, dev); -} - -extern int vio_ldc_send(struct vio_driver_state *vio, void *data, int len); -extern void vio_link_state_change(struct vio_driver_state *vio, int event); -extern void vio_conn_reset(struct vio_driver_state *vio); -extern int vio_control_pkt_engine(struct vio_driver_state *vio, void *pkt); -extern int vio_validate_sid(struct vio_driver_state *vio, - struct vio_msg_tag *tp); -extern u32 vio_send_sid(struct vio_driver_state *vio); -extern int vio_ldc_alloc(struct vio_driver_state *vio, - struct ldc_channel_config *base_cfg, void *event_arg); -extern void vio_ldc_free(struct vio_driver_state *vio); -extern int vio_driver_init(struct vio_driver_state *vio, struct vio_dev *vdev, - u8 dev_class, struct vio_version *ver_table, - int ver_table_size, struct vio_driver_ops *ops, - char *name); - -extern void vio_port_up(struct vio_driver_state *vio); +void vio_unregister_driver(struct vio_driver *drv); + +#define to_vio_driver(__drv) container_of_const(__drv, struct vio_driver, driver) +#define to_vio_dev(__dev) container_of_const(__dev, struct vio_dev, dev) + +int vio_ldc_send(struct vio_driver_state *vio, void *data, int len); +void vio_link_state_change(struct vio_driver_state *vio, int event); +void vio_conn_reset(struct vio_driver_state *vio); +int vio_control_pkt_engine(struct vio_driver_state *vio, void *pkt); +int vio_validate_sid(struct vio_driver_state *vio, + struct vio_msg_tag *tp); +u32 vio_send_sid(struct vio_driver_state *vio); +int vio_ldc_alloc(struct vio_driver_state *vio, + struct ldc_channel_config *base_cfg, void *event_arg); +void vio_ldc_free(struct vio_driver_state *vio); +int vio_driver_init(struct vio_driver_state *vio, struct vio_dev *vdev, + u8 dev_class, struct vio_version *ver_table, + int ver_table_size, struct vio_driver_ops *ops, + char *name); + +void vio_port_up(struct vio_driver_state *vio); +int vio_set_intr(unsigned long dev_ino, int state); +u64 vio_vdev_node(struct mdesc_handle *hp, struct vio_dev *vdev); #endif /* _SPARC64_VIO_H */ diff --git a/arch/sparc/include/asm/visasm.h b/arch/sparc/include/asm/visasm.h index 39ca301920db..71eb4e9afb3e 100644 --- a/arch/sparc/include/asm/visasm.h +++ b/arch/sparc/include/asm/visasm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_VISASM_H #define _SPARC64_VISASM_H @@ -28,21 +29,23 @@ * Must preserve %o5 between VISEntryHalf and VISExitHalf */ #define VISEntryHalf \ + VISEntry + +#define VISExitHalf \ + VISExit + +#define VISEntryHalfFast(fail_label) \ rd %fprs, %o5; \ andcc %o5, FPRS_FEF, %g0; \ be,pt %icc, 297f; \ - sethi %hi(298f), %g7; \ - sethi %hi(VISenterhalf), %g1; \ - jmpl %g1 + %lo(VISenterhalf), %g0; \ - or %g7, %lo(298f), %g7; \ - clr %o5; \ -297: wr %o5, FPRS_FEF, %fprs; \ -298: + nop; \ + ba,a,pt %xcc, fail_label; \ +297: wr %o5, FPRS_FEF, %fprs; -#define VISExitHalf \ +#define VISExitHalfFast \ wr %o5, 0, %fprs; -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ static inline void save_and_clear_fpu(void) { __asm__ __volatile__ ( " rd %%fprs, %%o5\n" @@ -57,7 +60,8 @@ static inline void save_and_clear_fpu(void) { " " : : "i" (FPRS_FEF|FPRS_DU) : "o5", "g1", "g2", "g3", "g7", "cc"); } -extern int vis_emul(struct pt_regs *, unsigned int); + +int vis_emul(struct pt_regs *, unsigned int); #endif #endif /* _SPARC64_ASI_H */ diff --git a/arch/sparc/include/asm/vmalloc.h b/arch/sparc/include/asm/vmalloc.h new file mode 100644 index 000000000000..04b8ab9518b8 --- /dev/null +++ b/arch/sparc/include/asm/vmalloc.h @@ -0,0 +1,4 @@ +#ifndef _ASM_SPARC_VMALLOC_H +#define _ASM_SPARC_VMALLOC_H + +#endif /* _ASM_SPARC_VMALLOC_H */ diff --git a/arch/sparc/include/asm/vvar.h b/arch/sparc/include/asm/vvar.h new file mode 100644 index 000000000000..6eaf5cfcaae1 --- /dev/null +++ b/arch/sparc/include/asm/vvar.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#ifndef _ASM_SPARC_VVAR_DATA_H +#define _ASM_SPARC_VVAR_DATA_H + +#include <asm/clocksource.h> +#include <asm/processor.h> +#include <asm/barrier.h> +#include <linux/time.h> +#include <linux/types.h> + +struct vvar_data { + unsigned int seq; + + int vclock_mode; + struct { /* extract of a clocksource struct */ + u64 cycle_last; + u64 mask; + int mult; + int shift; + } clock; + /* open coded 'struct timespec' */ + u64 wall_time_sec; + u64 wall_time_snsec; + u64 monotonic_time_snsec; + u64 monotonic_time_sec; + u64 monotonic_time_coarse_sec; + u64 monotonic_time_coarse_nsec; + u64 wall_time_coarse_sec; + u64 wall_time_coarse_nsec; + + int tz_minuteswest; + int tz_dsttime; +}; + +extern struct vvar_data *vvar_data; +extern int vdso_fix_stick; + +static inline unsigned int vvar_read_begin(const struct vvar_data *s) +{ + unsigned int ret; + +repeat: + ret = READ_ONCE(s->seq); + if (unlikely(ret & 1)) { + cpu_relax(); + goto repeat; + } + smp_rmb(); /* Finish all reads before we return seq */ + return ret; +} + +static inline int vvar_read_retry(const struct vvar_data *s, + unsigned int start) +{ + smp_rmb(); /* Finish all reads before checking the value of seq */ + return unlikely(s->seq != start); +} + +static inline void vvar_write_begin(struct vvar_data *s) +{ + ++s->seq; + smp_wmb(); /* Makes sure that increment of seq is reflected */ +} + +static inline void vvar_write_end(struct vvar_data *s) +{ + smp_wmb(); /* Makes the value of seq current before we increment */ + ++s->seq; +} + + +#endif /* _ASM_SPARC_VVAR_DATA_H */ diff --git a/arch/sparc/include/asm/winmacro.h b/arch/sparc/include/asm/winmacro.h index 9b7b21764cde..b6e911f5d93c 100644 --- a/arch/sparc/include/asm/winmacro.h +++ b/arch/sparc/include/asm/winmacro.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * winmacro.h: Window loading-unloading macros. * diff --git a/arch/sparc/include/asm/xor.h b/arch/sparc/include/asm/xor.h index 8ed591c7db2d..f4c651e203c4 100644 --- a/arch/sparc/include/asm/xor.h +++ b/arch/sparc/include/asm/xor.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_XOR_H #define ___ASM_SPARC_XOR_H #if defined(__sparc__) && defined(__arch64__) diff --git a/arch/sparc/include/asm/xor_32.h b/arch/sparc/include/asm/xor_32.h index 44bfa0787f3f..0351813cf3af 100644 --- a/arch/sparc/include/asm/xor_32.h +++ b/arch/sparc/include/asm/xor_32.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * include/asm/xor.h * * Optimized RAID-5 checksumming functions for 32-bit Sparc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * You should have received a copy of the GNU General Public License - * (for example /usr/src/linux/COPYING); if not, write to the Free - * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ /* @@ -21,7 +13,8 @@ */ static void -sparc_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) +sparc_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2) { int lines = bytes / (sizeof (long)) / 8; @@ -58,8 +51,9 @@ sparc_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) } static void -sparc_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3) +sparc_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) { int lines = bytes / (sizeof (long)) / 8; @@ -109,8 +103,10 @@ sparc_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -sparc_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4) +sparc_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) { int lines = bytes / (sizeof (long)) / 8; @@ -173,8 +169,11 @@ sparc_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, } static void -sparc_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, - unsigned long *p3, unsigned long *p4, unsigned long *p5) +sparc_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) { int lines = bytes / (sizeof (long)) / 8; diff --git a/arch/sparc/include/asm/xor_64.h b/arch/sparc/include/asm/xor_64.h index ee8edc68423e..caaddea8ad79 100644 --- a/arch/sparc/include/asm/xor_64.h +++ b/arch/sparc/include/asm/xor_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * include/asm/xor.h * @@ -7,26 +8,24 @@ * * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz) * Copyright (C) 2006 David S. Miller <davem@davemloft.net> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * You should have received a copy of the GNU General Public License - * (for example /usr/src/linux/COPYING); if not, write to the Free - * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #include <asm/spitfire.h> -extern void xor_vis_2(unsigned long, unsigned long *, unsigned long *); -extern void xor_vis_3(unsigned long, unsigned long *, unsigned long *, - unsigned long *); -extern void xor_vis_4(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *); -extern void xor_vis_5(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *, unsigned long *); +void xor_vis_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +void xor_vis_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +void xor_vis_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +void xor_vis_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); /* XXX Ugh, write cheetah versions... -DaveM */ @@ -38,13 +37,20 @@ static struct xor_block_template xor_block_VIS = { .do_5 = xor_vis_5, }; -extern void xor_niagara_2(unsigned long, unsigned long *, unsigned long *); -extern void xor_niagara_3(unsigned long, unsigned long *, unsigned long *, - unsigned long *); -extern void xor_niagara_4(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *); -extern void xor_niagara_5(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *, unsigned long *); +void xor_niagara_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); +void xor_niagara_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3); +void xor_niagara_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4); +void xor_niagara_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5); static struct xor_block_template xor_block_niagara = { .name = "Niagara", diff --git a/arch/sparc/include/uapi/asm/Kbuild b/arch/sparc/include/uapi/asm/Kbuild index b5843ee09fb5..353b70b1998f 100644 --- a/arch/sparc/include/uapi/asm/Kbuild +++ b/arch/sparc/include/uapi/asm/Kbuild @@ -1,50 +1,3 @@ -# UAPI Header export list -# User exported sparc header files - -include include/uapi/asm-generic/Kbuild.asm - -header-y += apc.h -header-y += asi.h -header-y += auxvec.h -header-y += bitsperlong.h -header-y += byteorder.h -header-y += display7seg.h -header-y += envctrl.h -header-y += errno.h -header-y += fbio.h -header-y += fcntl.h -header-y += ioctl.h -header-y += ioctls.h -header-y += ipcbuf.h -header-y += jsflash.h -header-y += kvm_para.h -header-y += mman.h -header-y += msgbuf.h -header-y += openpromio.h -header-y += param.h -header-y += perfctr.h -header-y += poll.h -header-y += posix_types.h -header-y += psr.h -header-y += psrcompat.h -header-y += pstate.h -header-y += ptrace.h -header-y += resource.h -header-y += sembuf.h -header-y += setup.h -header-y += shmbuf.h -header-y += sigcontext.h -header-y += siginfo.h -header-y += signal.h -header-y += socket.h -header-y += sockios.h -header-y += stat.h -header-y += statfs.h -header-y += swab.h -header-y += termbits.h -header-y += termios.h -header-y += traps.h -header-y += uctx.h -header-y += unistd.h -header-y += utrap.h -header-y += watchdog.h +# SPDX-License-Identifier: GPL-2.0 +generated-y += unistd_32.h +generated-y += unistd_64.h diff --git a/arch/sparc/include/uapi/asm/apc.h b/arch/sparc/include/uapi/asm/apc.h index 24e9a7d4d97e..aeb369b87fcd 100644 --- a/arch/sparc/include/uapi/asm/apc.h +++ b/arch/sparc/include/uapi/asm/apc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* apc - Driver definitions for power management functions * of Aurora Personality Chip (APC) on SPARCstation-4/5 and * derivatives diff --git a/arch/sparc/include/uapi/asm/asi.h b/arch/sparc/include/uapi/asm/asi.h index aace6f313716..fbb30a5b082f 100644 --- a/arch/sparc/include/uapi/asm/asi.h +++ b/arch/sparc/include/uapi/asm/asi.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _SPARC_ASI_H #define _SPARC_ASI_H @@ -144,6 +145,8 @@ * ASIs, "(4V)" designates SUN4V specific ASIs. "(NG4)" designates SPARC-T4 * and later ASIs. */ +#define ASI_MCD_PRIV_PRIMARY 0x02 /* (NG7) Privileged MCD version VA */ +#define ASI_MCD_REAL 0x05 /* (NG7) Privileged MCD version PA */ #define ASI_PHYS_USE_EC 0x14 /* PADDR, E-cachable */ #define ASI_PHYS_BYPASS_EC_E 0x15 /* PADDR, E-bit */ #define ASI_BLK_AIUP_4V 0x16 /* (4V) Prim, user, block ld/st */ @@ -244,6 +247,9 @@ #define ASI_UDBL_CONTROL_R 0x7f /* External UDB control regs rd low*/ #define ASI_INTR_R 0x7f /* IRQ vector dispatch read */ #define ASI_INTR_DATAN_R 0x7f /* (III) In irq vector data reg N */ +#define ASI_MCD_PRIMARY 0x90 /* (NG7) MCD version load/store */ +#define ASI_MCD_ST_BLKINIT_PRIMARY \ + 0x92 /* (NG7) MCD store BLKINIT primary */ #define ASI_PIC 0xb0 /* (NG4) PIC registers */ #define ASI_PST8_P 0xc0 /* Primary, 8 8-bit, partial */ #define ASI_PST8_S 0xc1 /* Secondary, 8 8-bit, partial */ @@ -279,7 +285,7 @@ * Most-Recently-Used, primary, * implicit */ -#define ASI_ST_BLKINIT_MRU_S 0xf2 /* (NG4) init-store, twin load, +#define ASI_ST_BLKINIT_MRU_S 0xf3 /* (NG4) init-store, twin load, * Most-Recently-Used, secondary, * implicit */ diff --git a/arch/sparc/include/uapi/asm/auxvec.h b/arch/sparc/include/uapi/asm/auxvec.h index ad6f360261f6..ab8780fb9df1 100644 --- a/arch/sparc/include/uapi/asm/auxvec.h +++ b/arch/sparc/include/uapi/asm/auxvec.h @@ -1,4 +1,15 @@ #ifndef __ASMSPARC_AUXVEC_H #define __ASMSPARC_AUXVEC_H +#define AT_SYSINFO_EHDR 33 + +/* Avoid overlap with other AT_* values since they are consolidated in + * glibc and any overlaps can cause problems + */ +#define AT_ADI_BLKSZ 48 +#define AT_ADI_NBITS 49 +#define AT_ADI_UEONADI 50 + +#define AT_VECTOR_SIZE_ARCH 4 + #endif /* !(__ASMSPARC_AUXVEC_H) */ diff --git a/arch/sparc/include/uapi/asm/bitsperlong.h b/arch/sparc/include/uapi/asm/bitsperlong.h index 40dcaa3aaa56..cd9a432278d2 100644 --- a/arch/sparc/include/uapi/asm/bitsperlong.h +++ b/arch/sparc/include/uapi/asm/bitsperlong.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef __ASM_ALPHA_BITSPERLONG_H #define __ASM_ALPHA_BITSPERLONG_H diff --git a/arch/sparc/include/uapi/asm/byteorder.h b/arch/sparc/include/uapi/asm/byteorder.h index ccc1b6b7de6c..216b8e59372f 100644 --- a/arch/sparc/include/uapi/asm/byteorder.h +++ b/arch/sparc/include/uapi/asm/byteorder.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _SPARC_BYTEORDER_H #define _SPARC_BYTEORDER_H diff --git a/arch/sparc/include/uapi/asm/display7seg.h b/arch/sparc/include/uapi/asm/display7seg.h index 86d4a901df24..7e9fef0c6efe 100644 --- a/arch/sparc/include/uapi/asm/display7seg.h +++ b/arch/sparc/include/uapi/asm/display7seg.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * * display7seg - Driver interface for the 7-segment display diff --git a/arch/sparc/include/uapi/asm/envctrl.h b/arch/sparc/include/uapi/asm/envctrl.h index 624fa7e2da8e..cf8aa0a14f40 100644 --- a/arch/sparc/include/uapi/asm/envctrl.h +++ b/arch/sparc/include/uapi/asm/envctrl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * * envctrl.h: Definitions for access to the i2c environment diff --git a/arch/sparc/include/uapi/asm/errno.h b/arch/sparc/include/uapi/asm/errno.h index c351aba997b7..81a732b902ee 100644 --- a/arch/sparc/include/uapi/asm/errno.h +++ b/arch/sparc/include/uapi/asm/errno.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _SPARC_ERRNO_H #define _SPARC_ERRNO_H @@ -40,7 +41,7 @@ #define EPROCLIM 67 /* SUNOS: Too many processes */ #define EUSERS 68 /* Too many users */ #define EDQUOT 69 /* Quota exceeded */ -#define ESTALE 70 /* Stale NFS file handle */ +#define ESTALE 70 /* Stale file handle */ #define EREMOTE 71 /* Object is remote */ #define ENOSTR 72 /* Device not a stream */ #define ETIME 73 /* Timer expired */ diff --git a/arch/sparc/include/uapi/asm/fbio.h b/arch/sparc/include/uapi/asm/fbio.h index d6cea07afb61..0dafe2c1eab7 100644 --- a/arch/sparc/include/uapi/asm/fbio.h +++ b/arch/sparc/include/uapi/asm/fbio.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI__LINUX_FBIO_H #define _UAPI__LINUX_FBIO_H diff --git a/arch/sparc/include/uapi/asm/fcntl.h b/arch/sparc/include/uapi/asm/fcntl.h index 7e8ace5bf760..67dae75e5274 100644 --- a/arch/sparc/include/uapi/asm/fcntl.h +++ b/arch/sparc/include/uapi/asm/fcntl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _SPARC_FCNTL_H #define _SPARC_FCNTL_H diff --git a/arch/sparc/include/uapi/asm/ioctl.h b/arch/sparc/include/uapi/asm/ioctl.h index 7d6bd51321b9..96c598fe746e 100644 --- a/arch/sparc/include/uapi/asm/ioctl.h +++ b/arch/sparc/include/uapi/asm/ioctl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _SPARC_IOCTL_H #define _SPARC_IOCTL_H diff --git a/arch/sparc/include/uapi/asm/ioctls.h b/arch/sparc/include/uapi/asm/ioctls.h index 897d1723fa14..7fd2f5873c9e 100644 --- a/arch/sparc/include/uapi/asm/ioctls.h +++ b/arch/sparc/include/uapi/asm/ioctls.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI_ASM_SPARC_IOCTLS_H #define _UAPI_ASM_SPARC_IOCTLS_H @@ -24,8 +25,12 @@ #define TIOCGPKT _IOR('T', 0x38, int) /* Get packet mode state */ #define TIOCGPTLCK _IOR('T', 0x39, int) /* Get Pty lock state */ #define TIOCGEXCL _IOR('T', 0x40, int) /* Get exclusive mode state */ +#define TIOCGRS485 _IOR('T', 0x41, struct serial_rs485) +#define TIOCSRS485 _IOWR('T', 0x42, struct serial_rs485) +#define TIOCGISO7816 _IOR('T', 0x43, struct serial_iso7816) +#define TIOCSISO7816 _IOWR('T', 0x44, struct serial_iso7816) -/* Note that all the ioctls that are not available in Linux have a +/* Note that all the ioctls that are not available in Linux have a * double underscore on the front to: a) avoid some programs to * think we support some ioctls under Linux (autoconfiguration stuff) */ @@ -86,6 +91,7 @@ #define TIOCGPTN _IOR('t', 134, unsigned int) /* Get Pty Number */ #define TIOCSPTLCK _IOW('t', 135, int) /* Lock/unlock PTY */ #define TIOCSIG _IOW('t', 136, int) /* Generate signal on Pty slave */ +#define TIOCGPTPEER _IO('t', 137) /* Safely open the slave */ /* Little f */ #define FIOCLEX _IO('f', 1) diff --git a/arch/sparc/include/uapi/asm/ipcbuf.h b/arch/sparc/include/uapi/asm/ipcbuf.h index 66013b4fe10d..0ea1240d2ea1 100644 --- a/arch/sparc/include/uapi/asm/ipcbuf.h +++ b/arch/sparc/include/uapi/asm/ipcbuf.h @@ -1,6 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef __SPARC_IPCBUF_H #define __SPARC_IPCBUF_H +#include <linux/posix_types.h> + /* * The ipc64_perm structure for sparc/sparc64 architecture. * Note extra padding because this structure is passed back and forth @@ -14,19 +17,19 @@ struct ipc64_perm { - __kernel_key_t key; - __kernel_uid_t uid; - __kernel_gid_t gid; - __kernel_uid_t cuid; - __kernel_gid_t cgid; + __kernel_key_t key; + __kernel_uid32_t uid; + __kernel_gid32_t gid; + __kernel_uid32_t cuid; + __kernel_gid32_t cgid; #ifndef __arch64__ - unsigned short __pad0; + unsigned short __pad0; #endif - __kernel_mode_t mode; - unsigned short __pad1; - unsigned short seq; - unsigned long long __unused1; - unsigned long long __unused2; + __kernel_mode_t mode; + unsigned short __pad1; + unsigned short seq; + unsigned long long __unused1; + unsigned long long __unused2; }; #endif /* __SPARC_IPCBUF_H */ diff --git a/arch/sparc/include/uapi/asm/jsflash.h b/arch/sparc/include/uapi/asm/jsflash.h deleted file mode 100644 index 0717d9e39d2d..000000000000 --- a/arch/sparc/include/uapi/asm/jsflash.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * jsflash.h: OS Flash SIMM support for JavaStations. - * - * Copyright (C) 1999 Pete Zaitcev - */ - -#ifndef _SPARC_JSFLASH_H -#define _SPARC_JSFLASH_H - -#ifndef _SPARC_TYPES_H -#include <linux/types.h> -#endif - -/* - * Semantics of the offset is a full address. - * Hardcode it or get it from probe ioctl. - * - * We use full bus address, so that we would be - * automatically compatible with possible future systems. - */ - -#define JSFLASH_IDENT (('F'<<8)|54) -struct jsflash_ident_arg { - __u64 off; /* 0x20000000 is included */ - __u32 size; - char name[32]; /* With trailing zero */ -}; - -#define JSFLASH_ERASE (('F'<<8)|55) -/* Put 0 as argument, may be flags or sector number... */ - -#define JSFLASH_PROGRAM (('F'<<8)|56) -struct jsflash_program_arg { - __u64 data; /* char* for sparc and sparc64 */ - __u64 off; - __u32 size; -}; - -#endif /* _SPARC_JSFLASH_H */ diff --git a/arch/sparc/include/uapi/asm/kvm_para.h b/arch/sparc/include/uapi/asm/kvm_para.h deleted file mode 100644 index 14fab8f0b957..000000000000 --- a/arch/sparc/include/uapi/asm/kvm_para.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm-generic/kvm_para.h> diff --git a/arch/sparc/include/uapi/asm/mman.h b/arch/sparc/include/uapi/asm/mman.h index 0b14df33cffa..cec9f4109687 100644 --- a/arch/sparc/include/uapi/asm/mman.h +++ b/arch/sparc/include/uapi/asm/mman.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI__SPARC_MMAN_H__ #define _UAPI__SPARC_MMAN_H__ @@ -5,6 +6,8 @@ /* SunOS'ified... */ +#define PROT_ADI 0x10 /* ADI enabled */ + #define MAP_RENAME MAP_ANONYMOUS /* In SunOS terminology */ #define MAP_NORESERVE 0x40 /* don't reserve swap pages */ #define MAP_INHERIT 0x80 /* SunOS doesn't do this, but... */ @@ -17,11 +20,6 @@ #define MCL_CURRENT 0x2000 /* lock all currently mapped pages */ #define MCL_FUTURE 0x4000 /* lock all additions to address space */ - -#define MAP_POPULATE 0x8000 /* populate (prefault) pagetables */ -#define MAP_NONBLOCK 0x10000 /* do not block on IO */ -#define MAP_STACK 0x20000 /* give out an address that is best suited for process/thread stacks */ -#define MAP_HUGETLB 0x40000 /* create a huge page mapping */ - +#define MCL_ONFAULT 0x8000 /* lock all pages that are faulted in */ #endif /* _UAPI__SPARC_MMAN_H__ */ diff --git a/arch/sparc/include/uapi/asm/msgbuf.h b/arch/sparc/include/uapi/asm/msgbuf.h index efc7cbe9788f..0954552da188 100644 --- a/arch/sparc/include/uapi/asm/msgbuf.h +++ b/arch/sparc/include/uapi/asm/msgbuf.h @@ -1,31 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _SPARC_MSGBUF_H #define _SPARC_MSGBUF_H +#include <asm/ipcbuf.h> + /* * The msqid64_ds structure for sparc64 architecture. * Note extra padding because this structure is passed back and forth * between kernel and user space. * * Pad space is left for: - * - 64-bit time_t to solve y2038 problem * - 2 miscellaneous 32-bit values */ - +struct msqid64_ds { + struct ipc64_perm msg_perm; #if defined(__sparc__) && defined(__arch64__) -# define PADDING(x) + long msg_stime; /* last msgsnd time */ + long msg_rtime; /* last msgrcv time */ + long msg_ctime; /* last change time */ #else -# define PADDING(x) unsigned int x; + unsigned long msg_stime_high; + unsigned long msg_stime; /* last msgsnd time */ + unsigned long msg_rtime_high; + unsigned long msg_rtime; /* last msgrcv time */ + unsigned long msg_ctime_high; + unsigned long msg_ctime; /* last change time */ #endif - - -struct msqid64_ds { - struct ipc64_perm msg_perm; - PADDING(__pad1) - __kernel_time_t msg_stime; /* last msgsnd time */ - PADDING(__pad2) - __kernel_time_t msg_rtime; /* last msgrcv time */ - PADDING(__pad3) - __kernel_time_t msg_ctime; /* last change time */ unsigned long msg_cbytes; /* current number of bytes on queue */ unsigned long msg_qnum; /* number of messages in queue */ unsigned long msg_qbytes; /* max number of bytes on queue */ @@ -34,5 +34,4 @@ struct msqid64_ds { unsigned long __unused1; unsigned long __unused2; }; -#undef PADDING #endif /* _SPARC_MSGBUF_H */ diff --git a/arch/sparc/include/uapi/asm/openpromio.h b/arch/sparc/include/uapi/asm/openpromio.h index 917fb8e9c633..2a73ec77aba6 100644 --- a/arch/sparc/include/uapi/asm/openpromio.h +++ b/arch/sparc/include/uapi/asm/openpromio.h @@ -1,19 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _SPARC_OPENPROMIO_H #define _SPARC_OPENPROMIO_H #include <linux/compiler.h> #include <linux/ioctl.h> -#include <linux/types.h> /* * SunOS and Solaris /dev/openprom definitions. The ioctl values * were chosen to be exactly equal to the SunOS equivalents. */ -struct openpromio -{ - u_int oprom_size; /* Actual size of the oprom_array. */ - char oprom_array[1]; /* Holds property names and values. */ +struct openpromio { + unsigned int oprom_size; /* Actual size of the oprom_array. */ + char oprom_array[]; /* Holds property names and values. */ }; #define OPROMMAXPARAM 4096 /* Maximum size of oprom_array. */ diff --git a/arch/sparc/include/uapi/asm/oradax.h b/arch/sparc/include/uapi/asm/oradax.h new file mode 100644 index 000000000000..0dace69058ab --- /dev/null +++ b/arch/sparc/include/uapi/asm/oradax.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later WITH Linux-syscall-note */ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + */ + +/* + * Oracle DAX driver API definitions + */ + +#ifndef _ORADAX_H +#define _ORADAX_H + +#include <linux/types.h> + +#define CCB_KILL 0 +#define CCB_INFO 1 +#define CCB_DEQUEUE 2 + +struct dax_command { + __u16 command; /* CCB_KILL/INFO/DEQUEUE */ + __u16 ca_offset; /* offset into mmapped completion area */ +}; + +struct ccb_kill_result { + __u16 action; /* action taken to kill ccb */ +}; + +struct ccb_info_result { + __u16 state; /* state of enqueued ccb */ + __u16 inst_num; /* dax instance number of enqueued ccb */ + __u16 q_num; /* queue number of enqueued ccb */ + __u16 q_pos; /* ccb position in queue */ +}; + +struct ccb_exec_result { + __u64 status_data; /* additional status data (e.g. bad VA) */ + __u32 status; /* one of DAX_SUBMIT_* */ +}; + +union ccb_result { + struct ccb_exec_result exec; + struct ccb_info_result info; + struct ccb_kill_result kill; +}; + +#define DAX_MMAP_LEN (16 * 1024) +#define DAX_MAX_CCBS 15 +#define DAX_CCB_BUF_MAXLEN (DAX_MAX_CCBS * 64) +#define DAX_NAME "oradax" + +/* CCB_EXEC status */ +#define DAX_SUBMIT_OK 0 +#define DAX_SUBMIT_ERR_RETRY 1 +#define DAX_SUBMIT_ERR_WOULDBLOCK 2 +#define DAX_SUBMIT_ERR_BUSY 3 +#define DAX_SUBMIT_ERR_THR_INIT 4 +#define DAX_SUBMIT_ERR_ARG_INVAL 5 +#define DAX_SUBMIT_ERR_CCB_INVAL 6 +#define DAX_SUBMIT_ERR_NO_CA_AVAIL 7 +#define DAX_SUBMIT_ERR_CCB_ARR_MMU_MISS 8 +#define DAX_SUBMIT_ERR_NOMAP 9 +#define DAX_SUBMIT_ERR_NOACCESS 10 +#define DAX_SUBMIT_ERR_TOOMANY 11 +#define DAX_SUBMIT_ERR_UNAVAIL 12 +#define DAX_SUBMIT_ERR_INTERNAL 13 + +/* CCB_INFO states - must match HV_CCB_STATE_* definitions */ +#define DAX_CCB_COMPLETED 0 +#define DAX_CCB_ENQUEUED 1 +#define DAX_CCB_INPROGRESS 2 +#define DAX_CCB_NOTFOUND 3 + +/* CCB_KILL actions - must match HV_CCB_KILL_* definitions */ +#define DAX_KILL_COMPLETED 0 +#define DAX_KILL_DEQUEUED 1 +#define DAX_KILL_KILLED 2 +#define DAX_KILL_NOTFOUND 3 + +#endif /* _ORADAX_H */ diff --git a/arch/sparc/include/uapi/asm/param.h b/arch/sparc/include/uapi/asm/param.h index 0bc356bf8c50..057d7135e4d0 100644 --- a/arch/sparc/include/uapi/asm/param.h +++ b/arch/sparc/include/uapi/asm/param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _ASMSPARC_PARAM_H #define _ASMSPARC_PARAM_H diff --git a/arch/sparc/include/uapi/asm/perfctr.h b/arch/sparc/include/uapi/asm/perfctr.h index 214feefa577c..316b837bcb8f 100644 --- a/arch/sparc/include/uapi/asm/perfctr.h +++ b/arch/sparc/include/uapi/asm/perfctr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /*---------------------------------------- PERFORMANCE INSTRUMENTATION Guillaume Thouvenin 08/10/98 diff --git a/arch/sparc/include/uapi/asm/poll.h b/arch/sparc/include/uapi/asm/poll.h index 091d3ad2e830..72356c999125 100644 --- a/arch/sparc/include/uapi/asm/poll.h +++ b/arch/sparc/include/uapi/asm/poll.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef __SPARC_POLL_H #define __SPARC_POLL_H diff --git a/arch/sparc/include/uapi/asm/posix_types.h b/arch/sparc/include/uapi/asm/posix_types.h index 156220ed99eb..f139e0048628 100644 --- a/arch/sparc/include/uapi/asm/posix_types.h +++ b/arch/sparc/include/uapi/asm/posix_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * This file is generally used by user-level software, so you need to * be a little careful about namespace pollution etc. Also, we cannot @@ -18,6 +19,16 @@ typedef unsigned short __kernel_old_gid_t; typedef int __kernel_suseconds_t; #define __kernel_suseconds_t __kernel_suseconds_t +typedef long __kernel_long_t; +typedef unsigned long __kernel_ulong_t; +#define __kernel_long_t __kernel_long_t + +struct __kernel_old_timeval { + __kernel_long_t tv_sec; + __kernel_suseconds_t tv_usec; +}; +#define __kernel_old_timeval __kernel_old_timeval + #else /* sparc 32 bit */ diff --git a/arch/sparc/include/uapi/asm/psr.h b/arch/sparc/include/uapi/asm/psr.h index 2f0ed856530b..e41f65f0aebd 100644 --- a/arch/sparc/include/uapi/asm/psr.h +++ b/arch/sparc/include/uapi/asm/psr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * psr.h: This file holds the macros for masking off various parts of * the processor status register on the Sparc. This is valid diff --git a/arch/sparc/include/uapi/asm/psrcompat.h b/arch/sparc/include/uapi/asm/psrcompat.h index 44b6327dbbf5..1eaffbe0d1e2 100644 --- a/arch/sparc/include/uapi/asm/psrcompat.h +++ b/arch/sparc/include/uapi/asm/psrcompat.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _SPARC64_PSRCOMPAT_H #define _SPARC64_PSRCOMPAT_H diff --git a/arch/sparc/include/uapi/asm/pstate.h b/arch/sparc/include/uapi/asm/pstate.h index 4b6b998afd99..ceca96e685c2 100644 --- a/arch/sparc/include/uapi/asm/pstate.h +++ b/arch/sparc/include/uapi/asm/pstate.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _SPARC64_PSTATE_H #define _SPARC64_PSTATE_H @@ -10,7 +11,12 @@ * ----------------------------------------------------------------------- * 63 12 11 10 9 8 7 6 5 4 3 2 1 0 */ +/* IG on V9 conflicts with MCDE on M7. PSTATE_MCDE will only be used on + * processors that support ADI which do not use IG, hence there is no + * functional conflict + */ #define PSTATE_IG _AC(0x0000000000000800,UL) /* Interrupt Globals. */ +#define PSTATE_MCDE _AC(0x0000000000000800,UL) /* MCD Enable */ #define PSTATE_MG _AC(0x0000000000000400,UL) /* MMU Globals. */ #define PSTATE_CLE _AC(0x0000000000000200,UL) /* Current Little Endian.*/ #define PSTATE_TLE _AC(0x0000000000000100,UL) /* Trap Little Endian. */ @@ -47,7 +53,12 @@ #define TSTATE_ASI _AC(0x00000000ff000000,UL) /* AddrSpace ID. */ #define TSTATE_PIL _AC(0x0000000000f00000,UL) /* %pil (Linux traps)*/ #define TSTATE_PSTATE _AC(0x00000000000fff00,UL) /* PSTATE. */ +/* IG on V9 conflicts with MCDE on M7. TSTATE_MCDE will only be used on + * processors that support ADI which do not support IG, hence there is + * no functional conflict + */ #define TSTATE_IG _AC(0x0000000000080000,UL) /* Interrupt Globals.*/ +#define TSTATE_MCDE _AC(0x0000000000080000,UL) /* MCD enable. */ #define TSTATE_MG _AC(0x0000000000040000,UL) /* MMU Globals. */ #define TSTATE_CLE _AC(0x0000000000020000,UL) /* CurrLittleEndian. */ #define TSTATE_TLE _AC(0x0000000000010000,UL) /* TrapLittleEndian. */ @@ -88,7 +99,7 @@ #define VERS_MAXTL _AC(0x000000000000ff00,UL) /* Max Trap Level. */ #define VERS_MAXWIN _AC(0x000000000000001f,UL) /* Max RegWindow Idx.*/ -/* Compatability Feature Register (%asr26), SPARC-T4 and later */ +/* Compatibility Feature Register (%asr26), SPARC-T4 and later */ #define CFR_AES _AC(0x0000000000000001,UL) /* Supports AES opcodes */ #define CFR_DES _AC(0x0000000000000002,UL) /* Supports DES opcodes */ #define CFR_KASUMI _AC(0x0000000000000004,UL) /* Supports KASUMI opcodes */ diff --git a/arch/sparc/include/uapi/asm/ptrace.h b/arch/sparc/include/uapi/asm/ptrace.h index 56fe4ea73feb..2eb677f4eb6a 100644 --- a/arch/sparc/include/uapi/asm/ptrace.h +++ b/arch/sparc/include/uapi/asm/ptrace.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI__SPARC_PTRACE_H #define _UAPI__SPARC_PTRACE_H @@ -14,7 +15,7 @@ */ #define PT_REGS_MAGIC 0x57ac6c00 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> @@ -87,7 +88,7 @@ struct sparc_trapf { unsigned long _unused; struct pt_regs *regs; }; -#endif /* (!__ASSEMBLY__) */ +#endif /* (!__ASSEMBLER__) */ #else /* 32 bit sparc */ @@ -96,7 +97,7 @@ struct sparc_trapf { /* This struct defines the way the registers are stored on the * stack during a system call and basically all traps. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> @@ -124,11 +125,11 @@ struct sparc_stackf { unsigned long xargs[6]; unsigned long xxargs[1]; }; -#endif /* (!__ASSEMBLY__) */ +#endif /* (!__ASSEMBLER__) */ #endif /* (defined(__sparc__) && defined(__arch64__))*/ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define TRACEREG_SZ sizeof(struct pt_regs) #define STACKFRAME_SZ sizeof(struct sparc_stackf) @@ -136,7 +137,7 @@ struct sparc_stackf { #define TRACEREG32_SZ sizeof(struct pt_regs32) #define STACKFRAME32_SZ sizeof(struct sparc_stackf32) -#endif /* (!__ASSEMBLY__) */ +#endif /* (!__ASSEMBLER__) */ #define UREG_G0 0 #define UREG_G1 1 @@ -160,30 +161,30 @@ struct sparc_stackf { #if defined(__sparc__) && defined(__arch64__) /* 64 bit sparc */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ -#else /* __ASSEMBLY__ */ +#else /* __ASSEMBLER__ */ /* For assembly code. */ #define TRACEREG_SZ 0xa0 #define STACKFRAME_SZ 0xc0 #define TRACEREG32_SZ 0x50 #define STACKFRAME32_SZ 0x60 -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #else /* (defined(__sparc__) && defined(__arch64__)) */ /* 32 bit sparc */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ -#else /* (!__ASSEMBLY__) */ +#else /* (!__ASSEMBLER__) */ /* For assembly code. */ #define TRACEREG_SZ 0x50 #define STACKFRAME_SZ 0x60 -#endif /* (!__ASSEMBLY__) */ +#endif /* (!__ASSEMBLER__) */ #endif /* (defined(__sparc__) && defined(__arch64__)) */ diff --git a/arch/sparc/include/uapi/asm/resource.h b/arch/sparc/include/uapi/asm/resource.h index fe163cafb4c7..cbe2de778838 100644 --- a/arch/sparc/include/uapi/asm/resource.h +++ b/arch/sparc/include/uapi/asm/resource.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * resource.h: Resource definitions. * diff --git a/arch/sparc/include/uapi/asm/sembuf.h b/arch/sparc/include/uapi/asm/sembuf.h index faee1be08d67..10621d066d79 100644 --- a/arch/sparc/include/uapi/asm/sembuf.h +++ b/arch/sparc/include/uapi/asm/sembuf.h @@ -1,31 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _SPARC_SEMBUF_H #define _SPARC_SEMBUF_H +#include <asm/ipcbuf.h> + /* * The semid64_ds structure for sparc architecture. * Note extra padding because this structure is passed back and forth * between kernel and user space. * * Pad space is left for: - * - 64-bit time_t to solve y2038 problem * - 2 miscellaneous 32-bit values */ -#if defined(__sparc__) && defined(__arch64__) -# define PADDING(x) -#else -# define PADDING(x) unsigned int x; -#endif struct semid64_ds { struct ipc64_perm sem_perm; /* permissions .. see ipc.h */ - PADDING(__pad1) - __kernel_time_t sem_otime; /* last semop time */ - PADDING(__pad2) - __kernel_time_t sem_ctime; /* last change time */ +#if defined(__sparc__) && defined(__arch64__) + long sem_otime; /* last semop time */ + long sem_ctime; /* last change time */ +#else + unsigned long sem_otime_high; + unsigned long sem_otime; /* last semop time */ + unsigned long sem_ctime_high; + unsigned long sem_ctime; /* last change time */ +#endif unsigned long sem_nsems; /* no. of semaphores in array */ unsigned long __unused1; unsigned long __unused2; }; -#undef PADDING #endif /* _SPARC64_SEMBUF_H */ diff --git a/arch/sparc/include/uapi/asm/setup.h b/arch/sparc/include/uapi/asm/setup.h index 533768450872..3c208a4dd464 100644 --- a/arch/sparc/include/uapi/asm/setup.h +++ b/arch/sparc/include/uapi/asm/setup.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * Just a place holder. */ diff --git a/arch/sparc/include/uapi/asm/shmbuf.h b/arch/sparc/include/uapi/asm/shmbuf.h index 83a16055363f..ed4f061c7a15 100644 --- a/arch/sparc/include/uapi/asm/shmbuf.h +++ b/arch/sparc/include/uapi/asm/shmbuf.h @@ -1,31 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _SPARC_SHMBUF_H #define _SPARC_SHMBUF_H +#include <asm/ipcbuf.h> +#include <asm/posix_types.h> + /* * The shmid64_ds structure for sparc architecture. * Note extra padding because this structure is passed back and forth * between kernel and user space. * * Pad space is left for: - * - 64-bit time_t to solve y2038 problem * - 2 miscellaneous 32-bit values */ +struct shmid64_ds { + struct ipc64_perm shm_perm; /* operation perms */ #if defined(__sparc__) && defined(__arch64__) -# define PADDING(x) + long shm_atime; /* last attach time */ + long shm_dtime; /* last detach time */ + long shm_ctime; /* last change time */ #else -# define PADDING(x) unsigned int x; + unsigned long shm_atime_high; + unsigned long shm_atime; /* last attach time */ + unsigned long shm_dtime_high; + unsigned long shm_dtime; /* last detach time */ + unsigned long shm_ctime_high; + unsigned long shm_ctime; /* last change time */ #endif - -struct shmid64_ds { - struct ipc64_perm shm_perm; /* operation perms */ - PADDING(__pad1) - __kernel_time_t shm_atime; /* last attach time */ - PADDING(__pad2) - __kernel_time_t shm_dtime; /* last detach time */ - PADDING(__pad3) - __kernel_time_t shm_ctime; /* last change time */ - size_t shm_segsz; /* size of segment (bytes) */ + __kernel_size_t shm_segsz; /* size of segment (bytes) */ __kernel_pid_t shm_cpid; /* pid of creator */ __kernel_pid_t shm_lpid; /* pid of last operator */ unsigned long shm_nattch; /* no. of current attaches */ @@ -45,6 +48,4 @@ struct shminfo64 { unsigned long __unused4; }; -#undef PADDING - #endif /* _SPARC_SHMBUF_H */ diff --git a/arch/sparc/include/uapi/asm/sigcontext.h b/arch/sparc/include/uapi/asm/sigcontext.h index ae5704fa77ad..043dd4b92026 100644 --- a/arch/sparc/include/uapi/asm/sigcontext.h +++ b/arch/sparc/include/uapi/asm/sigcontext.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * There isn't anything here anymore, but the file must not be empty or patch * will delete it. diff --git a/arch/sparc/include/uapi/asm/siginfo.h b/arch/sparc/include/uapi/asm/siginfo.h index 2d9b79ccaa50..0e7c27522aed 100644 --- a/arch/sparc/include/uapi/asm/siginfo.h +++ b/arch/sparc/include/uapi/asm/siginfo.h @@ -1,25 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI__SPARC_SIGINFO_H #define _UAPI__SPARC_SIGINFO_H #if defined(__sparc__) && defined(__arch64__) -#define __ARCH_SI_PREAMBLE_SIZE (4 * sizeof(int)) #define __ARCH_SI_BAND_T int #endif /* defined(__sparc__) && defined(__arch64__) */ - -#define __ARCH_SI_TRAPNO - #include <asm-generic/siginfo.h> #define SI_NOINFO 32767 /* no information in siginfo_t */ -/* - * SIGEMT si_codes - */ -#define EMT_TAGOVF (__SI_FAULT|1) /* tag overflow */ -#define NSIGEMT 1 - #endif /* _UAPI__SPARC_SIGINFO_H */ diff --git a/arch/sparc/include/uapi/asm/signal.h b/arch/sparc/include/uapi/asm/signal.h index f387400fcfdf..9c64d7cb85c2 100644 --- a/arch/sparc/include/uapi/asm/signal.h +++ b/arch/sparc/include/uapi/asm/signal.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI__SPARC_SIGNAL_H #define _UAPI__SPARC_SIGNAL_H @@ -104,7 +105,7 @@ #define __old_sigaction32 sigaction32 #endif -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ typedef unsigned long __old_sigset_t; /* at least 32 bits */ @@ -136,13 +137,11 @@ struct sigstack { #define SA_STACK _SV_SSTACK #define SA_ONSTACK _SV_SSTACK #define SA_RESTART _SV_INTR -#define SA_ONESHOT _SV_RESET +#define SA_RESETHAND _SV_RESET #define SA_NODEFER 0x20u #define SA_NOCLDWAIT 0x100u #define SA_SIGINFO 0x200u -#define SA_NOMASK SA_NODEFER - #define SIG_BLOCK 0x01 /* for blocking signals */ #define SIG_UNBLOCK 0x02 /* for unblocking signals */ #define SIG_SETMASK 0x04 /* for setting the signal mask */ @@ -152,6 +151,7 @@ struct sigstack { #include <asm-generic/signal-defs.h> +#include <asm/posix_types.h> #ifndef __KERNEL__ struct __new_sigaction { @@ -172,10 +172,10 @@ struct __old_sigaction { typedef struct sigaltstack { void __user *ss_sp; int ss_flags; - size_t ss_size; + __kernel_size_t ss_size; } stack_t; -#endif /* !(__ASSEMBLY__) */ +#endif /* !(__ASSEMBLER__) */ #endif /* _UAPI__SPARC_SIGNAL_H */ diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h index 4e1d66c3ce71..71befa109e1c 100644 --- a/arch/sparc/include/uapi/asm/socket.h +++ b/arch/sparc/include/uapi/asm/socket.h @@ -1,6 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _ASM_SOCKET_H #define _ASM_SOCKET_H +#include <linux/posix_types.h> #include <asm/sockios.h> /* For setsockopt(2) */ @@ -19,8 +21,8 @@ #define SO_BSDCOMPAT 0x0400 #define SO_RCVLOWAT 0x0800 #define SO_SNDLOWAT 0x1000 -#define SO_RCVTIMEO 0x2000 -#define SO_SNDTIMEO 0x4000 +#define SO_RCVTIMEO_OLD 0x2000 +#define SO_SNDTIMEO_OLD 0x4000 #define SO_ACCEPTCONN 0x8000 #define SO_SNDBUF 0x1001 @@ -32,7 +34,6 @@ #define SO_PROTOCOL 0x1028 #define SO_DOMAIN 0x1029 - /* Linux specific, keep the same. */ #define SO_NO_CHECK 0x000b #define SO_PRIORITY 0x000c @@ -44,19 +45,12 @@ #define SO_GET_FILTER SO_ATTACH_FILTER #define SO_PEERNAME 0x001c -#define SO_TIMESTAMP 0x001d -#define SCM_TIMESTAMP SO_TIMESTAMP #define SO_PEERSEC 0x001e #define SO_PASSSEC 0x001f -#define SO_TIMESTAMPNS 0x0021 -#define SCM_TIMESTAMPNS SO_TIMESTAMPNS #define SO_MARK 0x0022 -#define SO_TIMESTAMPING 0x0023 -#define SCM_TIMESTAMPING SO_TIMESTAMPING - #define SO_RXQ_OVFL 0x0024 #define SO_WIFI_STATUS 0x0025 @@ -72,9 +66,111 @@ #define SO_BUSY_POLL 0x0030 +#define SO_MAX_PACING_RATE 0x0031 + +#define SO_BPF_EXTENSIONS 0x0032 + +#define SO_INCOMING_CPU 0x0033 + +#define SO_ATTACH_BPF 0x0034 +#define SO_DETACH_BPF SO_DETACH_FILTER + +#define SO_ATTACH_REUSEPORT_CBPF 0x0035 +#define SO_ATTACH_REUSEPORT_EBPF 0x0036 + +#define SO_CNX_ADVICE 0x0037 + +#define SCM_TIMESTAMPING_OPT_STATS 0x0038 + +#define SO_MEMINFO 0x0039 + +#define SO_INCOMING_NAPI_ID 0x003a + +#define SO_COOKIE 0x003b + +#define SCM_TIMESTAMPING_PKTINFO 0x003c + +#define SO_PEERGROUPS 0x003d + +#define SO_ZEROCOPY 0x003e + +#define SO_TXTIME 0x003f +#define SCM_TXTIME SO_TXTIME + +#define SO_BINDTOIFINDEX 0x0041 + /* Security levels - as per NRL IPv6 - don't actually do anything */ #define SO_SECURITY_AUTHENTICATION 0x5001 #define SO_SECURITY_ENCRYPTION_TRANSPORT 0x5002 #define SO_SECURITY_ENCRYPTION_NETWORK 0x5004 +#define SO_TIMESTAMP_OLD 0x001d +#define SO_TIMESTAMPNS_OLD 0x0021 +#define SO_TIMESTAMPING_OLD 0x0023 + +#define SO_TIMESTAMP_NEW 0x0046 +#define SO_TIMESTAMPNS_NEW 0x0042 +#define SO_TIMESTAMPING_NEW 0x0043 + +#define SO_RCVTIMEO_NEW 0x0044 +#define SO_SNDTIMEO_NEW 0x0045 + +#define SO_DETACH_REUSEPORT_BPF 0x0047 + +#define SO_PREFER_BUSY_POLL 0x0048 +#define SO_BUSY_POLL_BUDGET 0x0049 + +#define SO_NETNS_COOKIE 0x0050 + +#define SO_BUF_LOCK 0x0051 + +#define SO_RESERVE_MEM 0x0052 + +#define SO_TXREHASH 0x0053 + +#define SO_RCVMARK 0x0054 + +#define SO_PASSPIDFD 0x0055 +#define SO_PEERPIDFD 0x0056 + +#define SO_DEVMEM_LINEAR 0x0057 +#define SCM_DEVMEM_LINEAR SO_DEVMEM_LINEAR +#define SO_DEVMEM_DMABUF 0x0058 +#define SCM_DEVMEM_DMABUF SO_DEVMEM_DMABUF +#define SO_DEVMEM_DONTNEED 0x0059 + +#define SCM_TS_OPT_ID 0x005a + +#define SO_RCVPRIORITY 0x005b + +#define SO_PASSRIGHTS 0x005c + +#define SO_INQ 0x005d +#define SCM_INQ SO_INQ + +#if !defined(__KERNEL__) + + +#if __BITS_PER_LONG == 64 +#define SO_TIMESTAMP SO_TIMESTAMP_OLD +#define SO_TIMESTAMPNS SO_TIMESTAMPNS_OLD +#define SO_TIMESTAMPING SO_TIMESTAMPING_OLD + +#define SO_RCVTIMEO SO_RCVTIMEO_OLD +#define SO_SNDTIMEO SO_SNDTIMEO_OLD +#else +#define SO_TIMESTAMP (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_TIMESTAMP_OLD : SO_TIMESTAMP_NEW) +#define SO_TIMESTAMPNS (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_TIMESTAMPNS_OLD : SO_TIMESTAMPNS_NEW) +#define SO_TIMESTAMPING (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_TIMESTAMPING_OLD : SO_TIMESTAMPING_NEW) + +#define SO_RCVTIMEO (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_RCVTIMEO_OLD : SO_RCVTIMEO_NEW) +#define SO_SNDTIMEO (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_SNDTIMEO_OLD : SO_SNDTIMEO_NEW) +#endif + +#define SCM_TIMESTAMP SO_TIMESTAMP +#define SCM_TIMESTAMPNS SO_TIMESTAMPNS +#define SCM_TIMESTAMPING SO_TIMESTAMPING + +#endif + #endif /* _ASM_SOCKET_H */ diff --git a/arch/sparc/include/uapi/asm/sockios.h b/arch/sparc/include/uapi/asm/sockios.h deleted file mode 100644 index 990ea746486b..000000000000 --- a/arch/sparc/include/uapi/asm/sockios.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef _ASM_SPARC_SOCKIOS_H -#define _ASM_SPARC_SOCKIOS_H - -/* Socket-level I/O control calls. */ -#define FIOSETOWN 0x8901 -#define SIOCSPGRP 0x8902 -#define FIOGETOWN 0x8903 -#define SIOCGPGRP 0x8904 -#define SIOCATMARK 0x8905 -#define SIOCGSTAMP 0x8906 /* Get stamp (timeval) */ -#define SIOCGSTAMPNS 0x8907 /* Get stamp (timespec) */ - -#endif /* !(_ASM_SPARC_SOCKIOS_H) */ - diff --git a/arch/sparc/include/uapi/asm/stat.h b/arch/sparc/include/uapi/asm/stat.h index a232e9e1f4e5..47f54133a141 100644 --- a/arch/sparc/include/uapi/asm/stat.h +++ b/arch/sparc/include/uapi/asm/stat.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef __SPARC_STAT_H #define __SPARC_STAT_H @@ -6,19 +7,19 @@ #if defined(__sparc__) && defined(__arch64__) /* 64 bit sparc */ struct stat { - unsigned st_dev; - ino_t st_ino; - mode_t st_mode; + unsigned int st_dev; + __kernel_ino_t st_ino; + __kernel_mode_t st_mode; short st_nlink; - uid_t st_uid; - gid_t st_gid; - unsigned st_rdev; - off_t st_size; - time_t st_atime; - time_t st_mtime; - time_t st_ctime; - off_t st_blksize; - off_t st_blocks; + __kernel_uid32_t st_uid; + __kernel_gid32_t st_gid; + unsigned int st_rdev; + long st_size; + long st_atime; + long st_mtime; + long st_ctime; + long st_blksize; + long st_blocks; unsigned long __unused4[2]; }; @@ -50,21 +51,21 @@ struct stat64 { /* 32 bit sparc */ struct stat { unsigned short st_dev; - ino_t st_ino; - mode_t st_mode; + __kernel_ino_t st_ino; + __kernel_mode_t st_mode; short st_nlink; unsigned short st_uid; unsigned short st_gid; unsigned short st_rdev; - off_t st_size; - time_t st_atime; + long st_size; + long st_atime; unsigned long st_atime_nsec; - time_t st_mtime; + long st_mtime; unsigned long st_mtime_nsec; - time_t st_ctime; + long st_ctime; unsigned long st_ctime_nsec; - off_t st_blksize; - off_t st_blocks; + long st_blksize; + long st_blocks; unsigned long __unused4[2]; }; diff --git a/arch/sparc/include/uapi/asm/statfs.h b/arch/sparc/include/uapi/asm/statfs.h deleted file mode 100644 index 55e607ad461d..000000000000 --- a/arch/sparc/include/uapi/asm/statfs.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef ___ASM_SPARC_STATFS_H -#define ___ASM_SPARC_STATFS_H - -#include <asm-generic/statfs.h> - -#endif diff --git a/arch/sparc/include/uapi/asm/swab.h b/arch/sparc/include/uapi/asm/swab.h index a34ad079487e..6b1b3f1ad725 100644 --- a/arch/sparc/include/uapi/asm/swab.h +++ b/arch/sparc/include/uapi/asm/swab.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _SPARC_SWAB_H #define _SPARC_SWAB_H @@ -9,9 +10,9 @@ static inline __u16 __arch_swab16p(const __u16 *addr) { __u16 ret; - __asm__ __volatile__ ("lduha [%1] %2, %0" + __asm__ __volatile__ ("lduha [%2] %3, %0" : "=r" (ret) - : "r" (addr), "i" (ASI_PL)); + : "m" (*addr), "r" (addr), "i" (ASI_PL)); return ret; } #define __arch_swab16p __arch_swab16p @@ -20,9 +21,9 @@ static inline __u32 __arch_swab32p(const __u32 *addr) { __u32 ret; - __asm__ __volatile__ ("lduwa [%1] %2, %0" + __asm__ __volatile__ ("lduwa [%2] %3, %0" : "=r" (ret) - : "r" (addr), "i" (ASI_PL)); + : "m" (*addr), "r" (addr), "i" (ASI_PL)); return ret; } #define __arch_swab32p __arch_swab32p @@ -31,9 +32,9 @@ static inline __u64 __arch_swab64p(const __u64 *addr) { __u64 ret; - __asm__ __volatile__ ("ldxa [%1] %2, %0" + __asm__ __volatile__ ("ldxa [%2] %3, %0" : "=r" (ret) - : "r" (addr), "i" (ASI_PL)); + : "m" (*addr), "r" (addr), "i" (ASI_PL)); return ret; } #define __arch_swab64p __arch_swab64p diff --git a/arch/sparc/include/uapi/asm/termbits.h b/arch/sparc/include/uapi/asm/termbits.h index dd91642fcca7..0da2b1adc0f5 100644 --- a/arch/sparc/include/uapi/asm/termbits.h +++ b/arch/sparc/include/uapi/asm/termbits.h @@ -1,27 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI_SPARC_TERMBITS_H #define _UAPI_SPARC_TERMBITS_H -#include <linux/posix_types.h> - -typedef unsigned char cc_t; -typedef unsigned int speed_t; +#include <asm-generic/termbits-common.h> #if defined(__sparc__) && defined(__arch64__) -typedef unsigned int tcflag_t; +typedef unsigned int tcflag_t; #else -typedef unsigned long tcflag_t; +typedef unsigned long tcflag_t; #endif -#define NCC 8 -struct termio { - unsigned short c_iflag; /* input mode flags */ - unsigned short c_oflag; /* output mode flags */ - unsigned short c_cflag; /* control mode flags */ - unsigned short c_lflag; /* local mode flags */ - unsigned char c_line; /* line discipline */ - unsigned char c_cc[NCC]; /* control characters */ -}; - #define NCCS 17 struct termios { tcflag_t c_iflag; /* input mode flags */ @@ -60,21 +48,19 @@ struct ktermios { }; /* c_cc characters */ -#define VINTR 0 -#define VQUIT 1 -#define VERASE 2 -#define VKILL 3 -#define VEOF 4 -#define VEOL 5 -#define VEOL2 6 -#define VSWTC 7 -#define VSTART 8 -#define VSTOP 9 - - +#define VINTR 0 +#define VQUIT 1 +#define VERASE 2 +#define VKILL 3 +#define VEOF 4 +#define VEOL 5 +#define VEOL2 6 +#define VSWTC 7 +#define VSTART 8 +#define VSTOP 9 #define VSUSP 10 -#define VDSUSP 11 /* SunOS POSIX nicety I do believe... */ +#define VDSUSP 11 /* SunOS POSIX nicety I do believe... */ #define VREPRINT 12 #define VDISCARD 13 #define VWERASE 14 @@ -89,121 +75,83 @@ struct ktermios { #endif /* c_iflag bits */ -#define IGNBRK 0x00000001 -#define BRKINT 0x00000002 -#define IGNPAR 0x00000004 -#define PARMRK 0x00000008 -#define INPCK 0x00000010 -#define ISTRIP 0x00000020 -#define INLCR 0x00000040 -#define IGNCR 0x00000080 -#define ICRNL 0x00000100 -#define IUCLC 0x00000200 -#define IXON 0x00000400 -#define IXANY 0x00000800 -#define IXOFF 0x00001000 -#define IMAXBEL 0x00002000 -#define IUTF8 0x00004000 +#define IUCLC 0x0200 +#define IXON 0x0400 +#define IXOFF 0x1000 +#define IMAXBEL 0x2000 +#define IUTF8 0x4000 /* c_oflag bits */ -#define OPOST 0x00000001 -#define OLCUC 0x00000002 -#define ONLCR 0x00000004 -#define OCRNL 0x00000008 -#define ONOCR 0x00000010 -#define ONLRET 0x00000020 -#define OFILL 0x00000040 -#define OFDEL 0x00000080 -#define NLDLY 0x00000100 -#define NL0 0x00000000 -#define NL1 0x00000100 -#define CRDLY 0x00000600 -#define CR0 0x00000000 -#define CR1 0x00000200 -#define CR2 0x00000400 -#define CR3 0x00000600 -#define TABDLY 0x00001800 -#define TAB0 0x00000000 -#define TAB1 0x00000800 -#define TAB2 0x00001000 -#define TAB3 0x00001800 -#define XTABS 0x00001800 -#define BSDLY 0x00002000 -#define BS0 0x00000000 -#define BS1 0x00002000 -#define VTDLY 0x00004000 -#define VT0 0x00000000 -#define VT1 0x00004000 -#define FFDLY 0x00008000 -#define FF0 0x00000000 -#define FF1 0x00008000 -#define PAGEOUT 0x00010000 /* SUNOS specific */ -#define WRAP 0x00020000 /* SUNOS specific */ +#define OLCUC 0x00002 +#define ONLCR 0x00004 +#define NLDLY 0x00100 +#define NL0 0x00000 +#define NL1 0x00100 +#define CRDLY 0x00600 +#define CR0 0x00000 +#define CR1 0x00200 +#define CR2 0x00400 +#define CR3 0x00600 +#define TABDLY 0x01800 +#define TAB0 0x00000 +#define TAB1 0x00800 +#define TAB2 0x01000 +#define TAB3 0x01800 +#define XTABS 0x01800 +#define BSDLY 0x02000 +#define BS0 0x00000 +#define BS1 0x02000 +#define VTDLY 0x04000 +#define VT0 0x00000 +#define VT1 0x04000 +#define FFDLY 0x08000 +#define FF0 0x00000 +#define FF1 0x08000 +#define PAGEOUT 0x10000 /* SUNOS specific */ +#define WRAP 0x20000 /* SUNOS specific */ /* c_cflag bit meaning */ -#define CBAUD 0x0000100f -#define B0 0x00000000 /* hang up */ -#define B50 0x00000001 -#define B75 0x00000002 -#define B110 0x00000003 -#define B134 0x00000004 -#define B150 0x00000005 -#define B200 0x00000006 -#define B300 0x00000007 -#define B600 0x00000008 -#define B1200 0x00000009 -#define B1800 0x0000000a -#define B2400 0x0000000b -#define B4800 0x0000000c -#define B9600 0x0000000d -#define B19200 0x0000000e -#define B38400 0x0000000f -#define EXTA B19200 -#define EXTB B38400 -#define CSIZE 0x00000030 -#define CS5 0x00000000 -#define CS6 0x00000010 -#define CS7 0x00000020 -#define CS8 0x00000030 -#define CSTOPB 0x00000040 -#define CREAD 0x00000080 -#define PARENB 0x00000100 -#define PARODD 0x00000200 -#define HUPCL 0x00000400 -#define CLOCAL 0x00000800 -#define CBAUDEX 0x00001000 +#define CBAUD 0x0000100f +#define CSIZE 0x00000030 +#define CS5 0x00000000 +#define CS6 0x00000010 +#define CS7 0x00000020 +#define CS8 0x00000030 +#define CSTOPB 0x00000040 +#define CREAD 0x00000080 +#define PARENB 0x00000100 +#define PARODD 0x00000200 +#define HUPCL 0x00000400 +#define CLOCAL 0x00000800 +#define CBAUDEX 0x00001000 /* We'll never see these speeds with the Zilogs, but for completeness... */ -#define BOTHER 0x00001000 -#define B57600 0x00001001 -#define B115200 0x00001002 -#define B230400 0x00001003 -#define B460800 0x00001004 +#define BOTHER 0x00001000 +#define B57600 0x00001001 +#define B115200 0x00001002 +#define B230400 0x00001003 +#define B460800 0x00001004 /* This is what we can do with the Zilogs. */ -#define B76800 0x00001005 +#define B76800 0x00001005 /* This is what we can do with the SAB82532. */ -#define B153600 0x00001006 -#define B307200 0x00001007 -#define B614400 0x00001008 -#define B921600 0x00001009 +#define B153600 0x00001006 +#define B307200 0x00001007 +#define B614400 0x00001008 +#define B921600 0x00001009 /* And these are the rest... */ -#define B500000 0x0000100a -#define B576000 0x0000100b -#define B1000000 0x0000100c -#define B1152000 0x0000100d -#define B1500000 0x0000100e -#define B2000000 0x0000100f +#define B500000 0x0000100a +#define B576000 0x0000100b +#define B1000000 0x0000100c +#define B1152000 0x0000100d +#define B1500000 0x0000100e +#define B2000000 0x0000100f /* These have totally bogus values and nobody uses them so far. Later on we'd have to use say 0x10000x and adjust CBAUD constant and drivers accordingly. -#define B2500000 0x00001010 -#define B3000000 0x00001011 -#define B3500000 0x00001012 -#define B4000000 0x00001013 */ -#define CIBAUD 0x100f0000 /* input baud rate (not used) */ -#define CMSPAR 0x40000000 /* mark or space (stick) parity */ -#define CRTSCTS 0x80000000 /* flow control */ - -#define IBSHIFT 16 /* Shift from CBAUD to CIBAUD */ +#define B2500000 0x00001010 +#define B3000000 0x00001011 +#define B3500000 0x00001012 +#define B4000000 0x00001013 */ +#define CIBAUD 0x100f0000 /* input baud rate (not used) */ /* c_lflag bits */ #define ISIG 0x00000001 @@ -218,7 +166,7 @@ struct ktermios { #define ECHOCTL 0x00000200 #define ECHOPRT 0x00000400 #define ECHOKE 0x00000800 -#define DEFECHO 0x00001000 /* SUNOS thing, what is it? */ +#define DEFECHO 0x00001000 /* SUNOS thing, what is it? */ #define FLUSHO 0x00002000 #define PENDIN 0x00004000 #define IEXTEN 0x00008000 @@ -243,21 +191,9 @@ struct ktermios { /* ioctl (fd, TIOCSERGETLSR, &result) where result may be as below */ #define TIOCSER_TEMT 0x01 /* Transmitter physically empty */ - -/* tcflow() and TCXONC use these */ -#define TCOOFF 0 -#define TCOON 1 -#define TCIOFF 2 -#define TCION 3 - -/* tcflush() and TCFLSH use these */ -#define TCIFLUSH 0 -#define TCOFLUSH 1 -#define TCIOFLUSH 2 - /* tcsetattr uses these */ -#define TCSANOW 0 -#define TCSADRAIN 1 -#define TCSAFLUSH 2 +#define TCSANOW 0 +#define TCSADRAIN 1 +#define TCSAFLUSH 2 #endif /* _UAPI_SPARC_TERMBITS_H */ diff --git a/arch/sparc/include/uapi/asm/termios.h b/arch/sparc/include/uapi/asm/termios.h index ea6f09e51e53..cceb32260881 100644 --- a/arch/sparc/include/uapi/asm/termios.h +++ b/arch/sparc/include/uapi/asm/termios.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI_SPARC_TERMIOS_H #define _UAPI_SPARC_TERMIOS_H @@ -39,5 +40,14 @@ struct winsize { unsigned short ws_ypixel; }; +#define NCC 8 +struct termio { + unsigned short c_iflag; /* input mode flags */ + unsigned short c_oflag; /* output mode flags */ + unsigned short c_cflag; /* control mode flags */ + unsigned short c_lflag; /* local mode flags */ + unsigned char c_line; /* line discipline */ + unsigned char c_cc[NCC]; /* control characters */ +}; #endif /* _UAPI_SPARC_TERMIOS_H */ diff --git a/arch/sparc/include/uapi/asm/traps.h b/arch/sparc/include/uapi/asm/traps.h index a4eceace6ccf..43fe5b8fe8be 100644 --- a/arch/sparc/include/uapi/asm/traps.h +++ b/arch/sparc/include/uapi/asm/traps.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * traps.h: Format of entries for the Sparc trap table. * @@ -9,8 +10,8 @@ #define NUM_SPARC_TRAPS 255 -#ifndef __ASSEMBLY__ -#endif /* !(__ASSEMBLY__) */ +#ifndef __ASSEMBLER__ +#endif /* !(__ASSEMBLER__) */ /* For patching the trap table at boot time, we need to know how to * form various common Sparc instructions. Thus these macros... diff --git a/arch/sparc/include/uapi/asm/uctx.h b/arch/sparc/include/uapi/asm/uctx.h index dc937c75ffdd..13a13198e04f 100644 --- a/arch/sparc/include/uapi/asm/uctx.h +++ b/arch/sparc/include/uapi/asm/uctx.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * uctx.h: Sparc64 {set,get}context() register state layouts. * diff --git a/arch/sparc/include/uapi/asm/unistd.h b/arch/sparc/include/uapi/asm/unistd.h index 62ced589bcf7..7f5d773b8cfc 100644 --- a/arch/sparc/include/uapi/asm/unistd.h +++ b/arch/sparc/include/uapi/asm/unistd.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * System calls under the Sparc. * @@ -20,409 +21,13 @@ #endif #endif -#define __NR_restart_syscall 0 /* Linux Specific */ -#define __NR_exit 1 /* Common */ -#define __NR_fork 2 /* Common */ -#define __NR_read 3 /* Common */ -#define __NR_write 4 /* Common */ -#define __NR_open 5 /* Common */ -#define __NR_close 6 /* Common */ -#define __NR_wait4 7 /* Common */ -#define __NR_creat 8 /* Common */ -#define __NR_link 9 /* Common */ -#define __NR_unlink 10 /* Common */ -#define __NR_execv 11 /* SunOS Specific */ -#define __NR_chdir 12 /* Common */ -#define __NR_chown 13 /* Common */ -#define __NR_mknod 14 /* Common */ -#define __NR_chmod 15 /* Common */ -#define __NR_lchown 16 /* Common */ -#define __NR_brk 17 /* Common */ -#define __NR_perfctr 18 /* Performance counter operations */ -#define __NR_lseek 19 /* Common */ -#define __NR_getpid 20 /* Common */ -#define __NR_capget 21 /* Linux Specific */ -#define __NR_capset 22 /* Linux Specific */ -#define __NR_setuid 23 /* Implemented via setreuid in SunOS */ -#define __NR_getuid 24 /* Common */ -#define __NR_vmsplice 25 /* ENOSYS under SunOS */ -#define __NR_ptrace 26 /* Common */ -#define __NR_alarm 27 /* Implemented via setitimer in SunOS */ -#define __NR_sigaltstack 28 /* Common */ -#define __NR_pause 29 /* Is sigblock(0)->sigpause() in SunOS */ -#define __NR_utime 30 /* Implemented via utimes() under SunOS */ -#ifdef __32bit_syscall_numbers__ -#define __NR_lchown32 31 /* Linux sparc32 specific */ -#define __NR_fchown32 32 /* Linux sparc32 specific */ -#endif -#define __NR_access 33 /* Common */ -#define __NR_nice 34 /* Implemented via get/setpriority() in SunOS */ -#ifdef __32bit_syscall_numbers__ -#define __NR_chown32 35 /* Linux sparc32 specific */ -#endif -#define __NR_sync 36 /* Common */ -#define __NR_kill 37 /* Common */ -#define __NR_stat 38 /* Common */ -#define __NR_sendfile 39 /* Linux Specific */ -#define __NR_lstat 40 /* Common */ -#define __NR_dup 41 /* Common */ -#define __NR_pipe 42 /* Common */ -#define __NR_times 43 /* Implemented via getrusage() in SunOS */ -#ifdef __32bit_syscall_numbers__ -#define __NR_getuid32 44 /* Linux sparc32 specific */ -#endif -#define __NR_umount2 45 /* Linux Specific */ -#define __NR_setgid 46 /* Implemented via setregid() in SunOS */ -#define __NR_getgid 47 /* Common */ -#define __NR_signal 48 /* Implemented via sigvec() in SunOS */ -#define __NR_geteuid 49 /* SunOS calls getuid() */ -#define __NR_getegid 50 /* SunOS calls getgid() */ -#define __NR_acct 51 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_getgid32 53 /* Linux sparc32 specific */ +#ifdef __arch64__ +#include <asm/unistd_64.h> #else -#define __NR_memory_ordering 52 /* Linux Specific */ -#endif -#define __NR_ioctl 54 /* Common */ -#define __NR_reboot 55 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_mmap2 56 /* Linux sparc32 Specific */ -#endif -#define __NR_symlink 57 /* Common */ -#define __NR_readlink 58 /* Common */ -#define __NR_execve 59 /* Common */ -#define __NR_umask 60 /* Common */ -#define __NR_chroot 61 /* Common */ -#define __NR_fstat 62 /* Common */ -#define __NR_fstat64 63 /* Linux Specific */ -#define __NR_getpagesize 64 /* Common */ -#define __NR_msync 65 /* Common in newer 1.3.x revs... */ -#define __NR_vfork 66 /* Common */ -#define __NR_pread64 67 /* Linux Specific */ -#define __NR_pwrite64 68 /* Linux Specific */ -#ifdef __32bit_syscall_numbers__ -#define __NR_geteuid32 69 /* Linux sparc32, sbrk under SunOS */ -#define __NR_getegid32 70 /* Linux sparc32, sstk under SunOS */ -#endif -#define __NR_mmap 71 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setreuid32 72 /* Linux sparc32, vadvise under SunOS */ -#endif -#define __NR_munmap 73 /* Common */ -#define __NR_mprotect 74 /* Common */ -#define __NR_madvise 75 /* Common */ -#define __NR_vhangup 76 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_truncate64 77 /* Linux sparc32 Specific */ -#endif -#define __NR_mincore 78 /* Common */ -#define __NR_getgroups 79 /* Common */ -#define __NR_setgroups 80 /* Common */ -#define __NR_getpgrp 81 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setgroups32 82 /* Linux sparc32, setpgrp under SunOS */ -#endif -#define __NR_setitimer 83 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_ftruncate64 84 /* Linux sparc32 Specific */ -#endif -#define __NR_swapon 85 /* Common */ -#define __NR_getitimer 86 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setuid32 87 /* Linux sparc32, gethostname under SunOS */ -#endif -#define __NR_sethostname 88 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setgid32 89 /* Linux sparc32, getdtablesize under SunOS */ -#endif -#define __NR_dup2 90 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setfsuid32 91 /* Linux sparc32, getdopt under SunOS */ -#endif -#define __NR_fcntl 92 /* Common */ -#define __NR_select 93 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setfsgid32 94 /* Linux sparc32, setdopt under SunOS */ +#include <asm/unistd_32.h> #endif -#define __NR_fsync 95 /* Common */ -#define __NR_setpriority 96 /* Common */ -#define __NR_socket 97 /* Common */ -#define __NR_connect 98 /* Common */ -#define __NR_accept 99 /* Common */ -#define __NR_getpriority 100 /* Common */ -#define __NR_rt_sigreturn 101 /* Linux Specific */ -#define __NR_rt_sigaction 102 /* Linux Specific */ -#define __NR_rt_sigprocmask 103 /* Linux Specific */ -#define __NR_rt_sigpending 104 /* Linux Specific */ -#define __NR_rt_sigtimedwait 105 /* Linux Specific */ -#define __NR_rt_sigqueueinfo 106 /* Linux Specific */ -#define __NR_rt_sigsuspend 107 /* Linux Specific */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setresuid32 108 /* Linux Specific, sigvec under SunOS */ -#define __NR_getresuid32 109 /* Linux Specific, sigblock under SunOS */ -#define __NR_setresgid32 110 /* Linux Specific, sigsetmask under SunOS */ -#define __NR_getresgid32 111 /* Linux Specific, sigpause under SunOS */ -#define __NR_setregid32 112 /* Linux sparc32, sigstack under SunOS */ -#else -#define __NR_setresuid 108 /* Linux Specific, sigvec under SunOS */ -#define __NR_getresuid 109 /* Linux Specific, sigblock under SunOS */ -#define __NR_setresgid 110 /* Linux Specific, sigsetmask under SunOS */ -#define __NR_getresgid 111 /* Linux Specific, sigpause under SunOS */ -#endif -#define __NR_recvmsg 113 /* Common */ -#define __NR_sendmsg 114 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_getgroups32 115 /* Linux sparc32, vtrace under SunOS */ -#endif -#define __NR_gettimeofday 116 /* Common */ -#define __NR_getrusage 117 /* Common */ -#define __NR_getsockopt 118 /* Common */ -#define __NR_getcwd 119 /* Linux Specific */ -#define __NR_readv 120 /* Common */ -#define __NR_writev 121 /* Common */ -#define __NR_settimeofday 122 /* Common */ -#define __NR_fchown 123 /* Common */ -#define __NR_fchmod 124 /* Common */ -#define __NR_recvfrom 125 /* Common */ -#define __NR_setreuid 126 /* Common */ -#define __NR_setregid 127 /* Common */ -#define __NR_rename 128 /* Common */ -#define __NR_truncate 129 /* Common */ -#define __NR_ftruncate 130 /* Common */ -#define __NR_flock 131 /* Common */ -#define __NR_lstat64 132 /* Linux Specific */ -#define __NR_sendto 133 /* Common */ -#define __NR_shutdown 134 /* Common */ -#define __NR_socketpair 135 /* Common */ -#define __NR_mkdir 136 /* Common */ -#define __NR_rmdir 137 /* Common */ -#define __NR_utimes 138 /* SunOS Specific */ -#define __NR_stat64 139 /* Linux Specific */ -#define __NR_sendfile64 140 /* adjtime under SunOS */ -#define __NR_getpeername 141 /* Common */ -#define __NR_futex 142 /* gethostid under SunOS */ -#define __NR_gettid 143 /* ENOSYS under SunOS */ -#define __NR_getrlimit 144 /* Common */ -#define __NR_setrlimit 145 /* Common */ -#define __NR_pivot_root 146 /* Linux Specific, killpg under SunOS */ -#define __NR_prctl 147 /* ENOSYS under SunOS */ -#define __NR_pciconfig_read 148 /* ENOSYS under SunOS */ -#define __NR_pciconfig_write 149 /* ENOSYS under SunOS */ -#define __NR_getsockname 150 /* Common */ -#define __NR_inotify_init 151 /* Linux specific */ -#define __NR_inotify_add_watch 152 /* Linux specific */ -#define __NR_poll 153 /* Common */ -#define __NR_getdents64 154 /* Linux specific */ -#ifdef __32bit_syscall_numbers__ -#define __NR_fcntl64 155 /* Linux sparc32 Specific */ -#endif -#define __NR_inotify_rm_watch 156 /* Linux specific */ -#define __NR_statfs 157 /* Common */ -#define __NR_fstatfs 158 /* Common */ -#define __NR_umount 159 /* Common */ -#define __NR_sched_set_affinity 160 /* Linux specific, async_daemon under SunOS */ -#define __NR_sched_get_affinity 161 /* Linux specific, getfh under SunOS */ -#define __NR_getdomainname 162 /* SunOS Specific */ -#define __NR_setdomainname 163 /* Common */ -#ifndef __32bit_syscall_numbers__ -#define __NR_utrap_install 164 /* SYSV ABI/v9 required */ -#endif -#define __NR_quotactl 165 /* Common */ -#define __NR_set_tid_address 166 /* Linux specific, exportfs under SunOS */ -#define __NR_mount 167 /* Common */ -#define __NR_ustat 168 /* Common */ -#define __NR_setxattr 169 /* SunOS: semsys */ -#define __NR_lsetxattr 170 /* SunOS: msgsys */ -#define __NR_fsetxattr 171 /* SunOS: shmsys */ -#define __NR_getxattr 172 /* SunOS: auditsys */ -#define __NR_lgetxattr 173 /* SunOS: rfssys */ -#define __NR_getdents 174 /* Common */ -#define __NR_setsid 175 /* Common */ -#define __NR_fchdir 176 /* Common */ -#define __NR_fgetxattr 177 /* SunOS: fchroot */ -#define __NR_listxattr 178 /* SunOS: vpixsys */ -#define __NR_llistxattr 179 /* SunOS: aioread */ -#define __NR_flistxattr 180 /* SunOS: aiowrite */ -#define __NR_removexattr 181 /* SunOS: aiowait */ -#define __NR_lremovexattr 182 /* SunOS: aiocancel */ -#define __NR_sigpending 183 /* Common */ -#define __NR_query_module 184 /* Linux Specific */ -#define __NR_setpgid 185 /* Common */ -#define __NR_fremovexattr 186 /* SunOS: pathconf */ -#define __NR_tkill 187 /* SunOS: fpathconf */ -#define __NR_exit_group 188 /* Linux specific, sysconf undef SunOS */ -#define __NR_uname 189 /* Linux Specific */ -#define __NR_init_module 190 /* Linux Specific */ -#define __NR_personality 191 /* Linux Specific */ -#define __NR_remap_file_pages 192 /* Linux Specific */ -#define __NR_epoll_create 193 /* Linux Specific */ -#define __NR_epoll_ctl 194 /* Linux Specific */ -#define __NR_epoll_wait 195 /* Linux Specific */ -#define __NR_ioprio_set 196 /* Linux Specific */ -#define __NR_getppid 197 /* Linux Specific */ -#define __NR_sigaction 198 /* Linux Specific */ -#define __NR_sgetmask 199 /* Linux Specific */ -#define __NR_ssetmask 200 /* Linux Specific */ -#define __NR_sigsuspend 201 /* Linux Specific */ -#define __NR_oldlstat 202 /* Linux Specific */ -#define __NR_uselib 203 /* Linux Specific */ -#define __NR_readdir 204 /* Linux Specific */ -#define __NR_readahead 205 /* Linux Specific */ -#define __NR_socketcall 206 /* Linux Specific */ -#define __NR_syslog 207 /* Linux Specific */ -#define __NR_lookup_dcookie 208 /* Linux Specific */ -#define __NR_fadvise64 209 /* Linux Specific */ -#define __NR_fadvise64_64 210 /* Linux Specific */ -#define __NR_tgkill 211 /* Linux Specific */ -#define __NR_waitpid 212 /* Linux Specific */ -#define __NR_swapoff 213 /* Linux Specific */ -#define __NR_sysinfo 214 /* Linux Specific */ -#define __NR_ipc 215 /* Linux Specific */ -#define __NR_sigreturn 216 /* Linux Specific */ -#define __NR_clone 217 /* Linux Specific */ -#define __NR_ioprio_get 218 /* Linux Specific */ -#define __NR_adjtimex 219 /* Linux Specific */ -#define __NR_sigprocmask 220 /* Linux Specific */ -#define __NR_create_module 221 /* Linux Specific */ -#define __NR_delete_module 222 /* Linux Specific */ -#define __NR_get_kernel_syms 223 /* Linux Specific */ -#define __NR_getpgid 224 /* Linux Specific */ -#define __NR_bdflush 225 /* Linux Specific */ -#define __NR_sysfs 226 /* Linux Specific */ -#define __NR_afs_syscall 227 /* Linux Specific */ -#define __NR_setfsuid 228 /* Linux Specific */ -#define __NR_setfsgid 229 /* Linux Specific */ -#define __NR__newselect 230 /* Linux Specific */ -#ifdef __32bit_syscall_numbers__ -#define __NR_time 231 /* Linux Specific */ -#else -#endif -#define __NR_splice 232 /* Linux Specific */ -#define __NR_stime 233 /* Linux Specific */ -#define __NR_statfs64 234 /* Linux Specific */ -#define __NR_fstatfs64 235 /* Linux Specific */ -#define __NR__llseek 236 /* Linux Specific */ -#define __NR_mlock 237 -#define __NR_munlock 238 -#define __NR_mlockall 239 -#define __NR_munlockall 240 -#define __NR_sched_setparam 241 -#define __NR_sched_getparam 242 -#define __NR_sched_setscheduler 243 -#define __NR_sched_getscheduler 244 -#define __NR_sched_yield 245 -#define __NR_sched_get_priority_max 246 -#define __NR_sched_get_priority_min 247 -#define __NR_sched_rr_get_interval 248 -#define __NR_nanosleep 249 -#define __NR_mremap 250 -#define __NR__sysctl 251 -#define __NR_getsid 252 -#define __NR_fdatasync 253 -#define __NR_nfsservctl 254 -#define __NR_sync_file_range 255 -#define __NR_clock_settime 256 -#define __NR_clock_gettime 257 -#define __NR_clock_getres 258 -#define __NR_clock_nanosleep 259 -#define __NR_sched_getaffinity 260 -#define __NR_sched_setaffinity 261 -#define __NR_timer_settime 262 -#define __NR_timer_gettime 263 -#define __NR_timer_getoverrun 264 -#define __NR_timer_delete 265 -#define __NR_timer_create 266 -/* #define __NR_vserver 267 Reserved for VSERVER */ -#define __NR_io_setup 268 -#define __NR_io_destroy 269 -#define __NR_io_submit 270 -#define __NR_io_cancel 271 -#define __NR_io_getevents 272 -#define __NR_mq_open 273 -#define __NR_mq_unlink 274 -#define __NR_mq_timedsend 275 -#define __NR_mq_timedreceive 276 -#define __NR_mq_notify 277 -#define __NR_mq_getsetattr 278 -#define __NR_waitid 279 -#define __NR_tee 280 -#define __NR_add_key 281 -#define __NR_request_key 282 -#define __NR_keyctl 283 -#define __NR_openat 284 -#define __NR_mkdirat 285 -#define __NR_mknodat 286 -#define __NR_fchownat 287 -#define __NR_futimesat 288 -#define __NR_fstatat64 289 -#define __NR_unlinkat 290 -#define __NR_renameat 291 -#define __NR_linkat 292 -#define __NR_symlinkat 293 -#define __NR_readlinkat 294 -#define __NR_fchmodat 295 -#define __NR_faccessat 296 -#define __NR_pselect6 297 -#define __NR_ppoll 298 -#define __NR_unshare 299 -#define __NR_set_robust_list 300 -#define __NR_get_robust_list 301 -#define __NR_migrate_pages 302 -#define __NR_mbind 303 -#define __NR_get_mempolicy 304 -#define __NR_set_mempolicy 305 -#define __NR_kexec_load 306 -#define __NR_move_pages 307 -#define __NR_getcpu 308 -#define __NR_epoll_pwait 309 -#define __NR_utimensat 310 -#define __NR_signalfd 311 -#define __NR_timerfd_create 312 -#define __NR_eventfd 313 -#define __NR_fallocate 314 -#define __NR_timerfd_settime 315 -#define __NR_timerfd_gettime 316 -#define __NR_signalfd4 317 -#define __NR_eventfd2 318 -#define __NR_epoll_create1 319 -#define __NR_dup3 320 -#define __NR_pipe2 321 -#define __NR_inotify_init1 322 -#define __NR_accept4 323 -#define __NR_preadv 324 -#define __NR_pwritev 325 -#define __NR_rt_tgsigqueueinfo 326 -#define __NR_perf_event_open 327 -#define __NR_recvmmsg 328 -#define __NR_fanotify_init 329 -#define __NR_fanotify_mark 330 -#define __NR_prlimit64 331 -#define __NR_name_to_handle_at 332 -#define __NR_open_by_handle_at 333 -#define __NR_clock_adjtime 334 -#define __NR_syncfs 335 -#define __NR_sendmmsg 336 -#define __NR_setns 337 -#define __NR_process_vm_readv 338 -#define __NR_process_vm_writev 339 -#define __NR_kern_features 340 -#define __NR_kcmp 341 -#define __NR_finit_module 342 - -#define NR_syscalls 343 /* Bitmask values returned from kern_features system call. */ #define KERN_FEATURE_MIXED_MODE_STACK 0x00000001 -#ifdef __32bit_syscall_numbers__ -/* Sparc 32-bit only has the "setresuid32", "getresuid32" variants, - * it never had the plain ones and there is no value to adding those - * old versions into the syscall table. - */ -#define __IGNORE_setresuid -#define __IGNORE_getresuid -#define __IGNORE_setresgid -#define __IGNORE_getresgid -#endif - #endif /* _UAPI_SPARC_UNISTD_H */ diff --git a/arch/sparc/include/uapi/asm/utrap.h b/arch/sparc/include/uapi/asm/utrap.h index b10e527c22d9..a489b08b6a33 100644 --- a/arch/sparc/include/uapi/asm/utrap.h +++ b/arch/sparc/include/uapi/asm/utrap.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * include/asm/utrap.h * @@ -43,9 +44,9 @@ #define UTH_NOCHANGE (-1) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ typedef int utrap_entry_t; typedef void *utrap_handler_t; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* !(__ASM_SPARC64_PROCESSOR_H) */ diff --git a/arch/sparc/include/uapi/asm/watchdog.h b/arch/sparc/include/uapi/asm/watchdog.h index 5baf2d3919cf..497ac19a9e4e 100644 --- a/arch/sparc/include/uapi/asm/watchdog.h +++ b/arch/sparc/include/uapi/asm/watchdog.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * * watchdog - Driver interface for the hardware watchdog timers diff --git a/arch/sparc/kernel/.gitignore b/arch/sparc/kernel/.gitignore index c5f676c3c224..bbb90f92d051 100644 --- a/arch/sparc/kernel/.gitignore +++ b/arch/sparc/kernel/.gitignore @@ -1 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only vmlinux.lds diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile index d432fb20358e..22170d4f8e06 100644 --- a/arch/sparc/kernel/Makefile +++ b/arch/sparc/kernel/Makefile @@ -1,16 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 + # # Makefile for the linux kernel. # -asflags-y := -ansi -ccflags-y := -Werror - -extra-y := head_$(BITS).o - # Undefine sparc when processing vmlinux.lds - it is used # And teach CPP we are doing $(BITS) builds (for this case) CPPFLAGS_vmlinux.lds := -Usparc -m$(BITS) -extra-y += vmlinux.lds +always-$(KBUILD_BUILTIN) += vmlinux.lds ifdef CONFIG_FUNCTION_TRACER # Do not profile debug and lowlevel utilities @@ -20,6 +17,8 @@ CFLAGS_REMOVE_perf_event.o := -pg CFLAGS_REMOVE_pcr.o := -pg endif +obj-y := head_$(BITS).o +obj-$(CONFIG_SPARC64) += urtt_fill.o obj-$(CONFIG_SPARC32) += entry.o wof.o wuf.o obj-$(CONFIG_SPARC32) += etrap_32.o obj-$(CONFIG_SPARC32) += rtrap_32.o @@ -30,9 +29,11 @@ obj-y += irq_$(BITS).o obj-$(CONFIG_SPARC32) += sun4m_irq.o sun4d_irq.o obj-y += process_$(BITS).o +obj-y += process.o obj-y += signal_$(BITS).o obj-y += sigutil_$(BITS).o obj-$(CONFIG_SPARC32) += ioport.o +obj-y += setup.o obj-y += setup_$(BITS).o obj-y += idprom.o obj-y += sys_sparc_$(BITS).o @@ -40,8 +41,8 @@ obj-$(CONFIG_SPARC32) += systbls_32.o obj-y += time_$(BITS).o obj-$(CONFIG_SPARC32) += windows.o obj-y += cpu.o +obj-$(CONFIG_SPARC64) += vdso.o obj-$(CONFIG_SPARC32) += devices.o -obj-$(CONFIG_SPARC32) += tadpole.o obj-y += ptrace_$(BITS).o obj-y += unaligned_$(BITS).o obj-y += una_asm_$(BITS).o @@ -56,7 +57,7 @@ obj-$(CONFIG_SPARC32) += leon_pmc.o obj-$(CONFIG_SPARC64) += reboot.o obj-$(CONFIG_SPARC64) += sysfs.o -obj-$(CONFIG_SPARC64) += iommu.o +obj-$(CONFIG_SPARC64) += iommu.o iommu-common.o obj-$(CONFIG_SPARC64) += central.o obj-$(CONFIG_SPARC64) += starfire.o obj-$(CONFIG_SPARC64) += power.o @@ -66,12 +67,11 @@ obj-$(CONFIG_SPARC64) += visemul.o obj-$(CONFIG_SPARC64) += hvapi.o obj-$(CONFIG_SPARC64) += sstate.o obj-$(CONFIG_SPARC64) += mdesc.o +obj-$(CONFIG_SPARC64) += adi_64.o obj-$(CONFIG_SPARC64) += pcr.o obj-$(CONFIG_SPARC64) += nmi.o obj-$(CONFIG_SPARC64_SMP) += cpumap.o -obj-y += dma.o - obj-$(CONFIG_PCIC_PCI) += pcic.o obj-$(CONFIG_LEON_PCI) += leon_pci.o obj-$(CONFIG_SPARC_GRPCI2)+= leon_pci_grpci2.o @@ -84,12 +84,13 @@ obj-$(CONFIG_SPARC64_SMP) += hvtramp.o obj-y += auxio_$(BITS).o obj-$(CONFIG_SUN_PM) += apc.o pmc.o +obj-y += termios.o + obj-$(CONFIG_MODULES) += module.o -obj-$(CONFIG_MODULES) += sparc_ksyms_$(BITS).o +obj-$(CONFIG_MODULES) += sparc_ksyms.o obj-$(CONFIG_SPARC_LED) += led.o obj-$(CONFIG_KGDB) += kgdb_$(BITS).o - obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o @@ -99,7 +100,8 @@ obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-$(CONFIG_SPARC64_PCI) += pci.o pci_common.o psycho_common.o obj-$(CONFIG_SPARC64_PCI) += pci_psycho.o pci_sabre.o pci_schizo.o obj-$(CONFIG_SPARC64_PCI) += pci_sun4v.o pci_sun4v_asm.o pci_fire.o -obj-$(CONFIG_PCI_MSI) += pci_msi.o +obj-$(CONFIG_SPARC64_PCI_MSI) += pci_msi.o + obj-$(CONFIG_COMPAT) += sys32.o sys_sparc32.o signal32.o @@ -115,4 +117,5 @@ obj-$(CONFIG_COMPAT) += $(audit--y) pc--$(CONFIG_PERF_EVENTS) := perf_event.o obj-$(CONFIG_SPARC64) += $(pc--y) -obj-$(CONFIG_SPARC64) += jump_label.o +obj-$(CONFIG_UPROBES) += uprobes.o +obj-$(CONFIG_JUMP_LABEL) += jump_label.o diff --git a/arch/sparc/kernel/adi_64.c b/arch/sparc/kernel/adi_64.c new file mode 100644 index 000000000000..18036a43cf56 --- /dev/null +++ b/arch/sparc/kernel/adi_64.c @@ -0,0 +1,396 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* adi_64.c: support for ADI (Application Data Integrity) feature on + * sparc m7 and newer processors. This feature is also known as + * SSM (Silicon Secured Memory). + * + * Copyright (C) 2016 Oracle and/or its affiliates. All rights reserved. + * Author: Khalid Aziz (khalid.aziz@oracle.com) + */ +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/mm_types.h> +#include <asm/mdesc.h> +#include <asm/adi_64.h> +#include <asm/mmu_64.h> +#include <asm/pgtable_64.h> + +/* Each page of storage for ADI tags can accommodate tags for 128 + * pages. When ADI enabled pages are being swapped out, it would be + * prudent to allocate at least enough tag storage space to accommodate + * SWAPFILE_CLUSTER number of pages. Allocate enough tag storage to + * store tags for four SWAPFILE_CLUSTER pages to reduce need for + * further allocations for same vma. + */ +#define TAG_STORAGE_PAGES 8 + +struct adi_config adi_state; +EXPORT_SYMBOL(adi_state); + +/* mdesc_adi_init() : Parse machine description provided by the + * hypervisor to detect ADI capabilities + * + * Hypervisor reports ADI capabilities of platform in "hwcap-list" property + * for "cpu" node. If the platform supports ADI, "hwcap-list" property + * contains the keyword "adp". If the platform supports ADI, "platform" + * node will contain "adp-blksz", "adp-nbits" and "ue-on-adp" properties + * to describe the ADI capabilities. + */ +void __init mdesc_adi_init(void) +{ + struct mdesc_handle *hp = mdesc_grab(); + const char *prop; + u64 pn, *val; + int len; + + if (!hp) + goto adi_not_found; + + pn = mdesc_node_by_name(hp, MDESC_NODE_NULL, "cpu"); + if (pn == MDESC_NODE_NULL) + goto adi_not_found; + + prop = mdesc_get_property(hp, pn, "hwcap-list", &len); + if (!prop) + goto adi_not_found; + + /* + * Look for "adp" keyword in hwcap-list which would indicate + * ADI support + */ + adi_state.enabled = false; + while (len) { + int plen; + + if (!strcmp(prop, "adp")) { + adi_state.enabled = true; + break; + } + + plen = strlen(prop) + 1; + prop += plen; + len -= plen; + } + + if (!adi_state.enabled) + goto adi_not_found; + + /* Find the ADI properties in "platform" node. If all ADI + * properties are not found, ADI support is incomplete and + * do not enable ADI in the kernel. + */ + pn = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform"); + if (pn == MDESC_NODE_NULL) + goto adi_not_found; + + val = (u64 *) mdesc_get_property(hp, pn, "adp-blksz", &len); + if (!val) + goto adi_not_found; + adi_state.caps.blksz = *val; + + val = (u64 *) mdesc_get_property(hp, pn, "adp-nbits", &len); + if (!val) + goto adi_not_found; + adi_state.caps.nbits = *val; + + val = (u64 *) mdesc_get_property(hp, pn, "ue-on-adp", &len); + if (!val) + goto adi_not_found; + adi_state.caps.ue_on_adi = *val; + + /* Some of the code to support swapping ADI tags is written + * assumption that two ADI tags can fit inside one byte. If + * this assumption is broken by a future architecture change, + * that code will have to be revisited. If that were to happen, + * disable ADI support so we do not get unpredictable results + * with programs trying to use ADI and their pages getting + * swapped out + */ + if (adi_state.caps.nbits > 4) { + pr_warn("WARNING: ADI tag size >4 on this platform. Disabling AADI support\n"); + adi_state.enabled = false; + } + + mdesc_release(hp); + return; + +adi_not_found: + adi_state.enabled = false; + adi_state.caps.blksz = 0; + adi_state.caps.nbits = 0; + if (hp) + mdesc_release(hp); +} + +static tag_storage_desc_t *find_tag_store(struct mm_struct *mm, + struct vm_area_struct *vma, + unsigned long addr) +{ + tag_storage_desc_t *tag_desc = NULL; + unsigned long i, max_desc, flags; + + /* Check if this vma already has tag storage descriptor + * allocated for it. + */ + max_desc = PAGE_SIZE/sizeof(tag_storage_desc_t); + if (mm->context.tag_store) { + tag_desc = mm->context.tag_store; + spin_lock_irqsave(&mm->context.tag_lock, flags); + for (i = 0; i < max_desc; i++) { + if ((addr >= tag_desc->start) && + ((addr + PAGE_SIZE - 1) <= tag_desc->end)) + break; + tag_desc++; + } + spin_unlock_irqrestore(&mm->context.tag_lock, flags); + + /* If no matching entries were found, this must be a + * freshly allocated page + */ + if (i >= max_desc) + tag_desc = NULL; + } + + return tag_desc; +} + +static tag_storage_desc_t *alloc_tag_store(struct mm_struct *mm, + struct vm_area_struct *vma, + unsigned long addr) +{ + unsigned char *tags; + unsigned long i, size, max_desc, flags; + tag_storage_desc_t *tag_desc, *open_desc; + unsigned long end_addr, hole_start, hole_end; + + max_desc = PAGE_SIZE/sizeof(tag_storage_desc_t); + open_desc = NULL; + hole_start = 0; + hole_end = ULONG_MAX; + end_addr = addr + PAGE_SIZE - 1; + + /* Check if this vma already has tag storage descriptor + * allocated for it. + */ + spin_lock_irqsave(&mm->context.tag_lock, flags); + if (mm->context.tag_store) { + tag_desc = mm->context.tag_store; + + /* Look for a matching entry for this address. While doing + * that, look for the first open slot as well and find + * the hole in already allocated range where this request + * will fit in. + */ + for (i = 0; i < max_desc; i++) { + if (tag_desc->tag_users == 0) { + if (open_desc == NULL) + open_desc = tag_desc; + } else { + if ((addr >= tag_desc->start) && + (tag_desc->end >= (addr + PAGE_SIZE - 1))) { + tag_desc->tag_users++; + goto out; + } + } + if ((tag_desc->start > end_addr) && + (tag_desc->start < hole_end)) + hole_end = tag_desc->start; + if ((tag_desc->end < addr) && + (tag_desc->end > hole_start)) + hole_start = tag_desc->end; + tag_desc++; + } + + } else { + size = sizeof(tag_storage_desc_t)*max_desc; + mm->context.tag_store = kzalloc(size, GFP_NOWAIT); + if (mm->context.tag_store == NULL) { + tag_desc = NULL; + goto out; + } + tag_desc = mm->context.tag_store; + for (i = 0; i < max_desc; i++, tag_desc++) + tag_desc->tag_users = 0; + open_desc = mm->context.tag_store; + i = 0; + } + + /* Check if we ran out of tag storage descriptors */ + if (open_desc == NULL) { + tag_desc = NULL; + goto out; + } + + /* Mark this tag descriptor slot in use and then initialize it */ + tag_desc = open_desc; + tag_desc->tag_users = 1; + + /* Tag storage has not been allocated for this vma and space + * is available in tag storage descriptor. Since this page is + * being swapped out, there is high probability subsequent pages + * in the VMA will be swapped out as well. Allocate pages to + * store tags for as many pages in this vma as possible but not + * more than TAG_STORAGE_PAGES. Each byte in tag space holds + * two ADI tags since each ADI tag is 4 bits. Each ADI tag + * covers adi_blksize() worth of addresses. Check if the hole is + * big enough to accommodate full address range for using + * TAG_STORAGE_PAGES number of tag pages. + */ + size = TAG_STORAGE_PAGES * PAGE_SIZE; + end_addr = addr + (size*2*adi_blksize()) - 1; + /* Check for overflow. If overflow occurs, allocate only one page */ + if (end_addr < addr) { + size = PAGE_SIZE; + end_addr = addr + (size*2*adi_blksize()) - 1; + /* If overflow happens with the minimum tag storage + * allocation as well, adjust ending address for this + * tag storage. + */ + if (end_addr < addr) + end_addr = ULONG_MAX; + } + if (hole_end < end_addr) { + /* Available hole is too small on the upper end of + * address. Can we expand the range towards the lower + * address and maximize use of this slot? + */ + unsigned long tmp_addr; + + end_addr = hole_end - 1; + tmp_addr = end_addr - (size*2*adi_blksize()) + 1; + /* Check for underflow. If underflow occurs, allocate + * only one page for storing ADI tags + */ + if (tmp_addr > addr) { + size = PAGE_SIZE; + tmp_addr = end_addr - (size*2*adi_blksize()) - 1; + /* If underflow happens with the minimum tag storage + * allocation as well, adjust starting address for + * this tag storage. + */ + if (tmp_addr > addr) + tmp_addr = 0; + } + if (tmp_addr < hole_start) { + /* Available hole is restricted on lower address + * end as well + */ + tmp_addr = hole_start + 1; + } + addr = tmp_addr; + size = (end_addr + 1 - addr)/(2*adi_blksize()); + size = (size + (PAGE_SIZE-adi_blksize()))/PAGE_SIZE; + size = size * PAGE_SIZE; + } + tags = kzalloc(size, GFP_NOWAIT); + if (tags == NULL) { + tag_desc->tag_users = 0; + tag_desc = NULL; + goto out; + } + tag_desc->start = addr; + tag_desc->tags = tags; + tag_desc->end = end_addr; + +out: + spin_unlock_irqrestore(&mm->context.tag_lock, flags); + return tag_desc; +} + +static void del_tag_store(tag_storage_desc_t *tag_desc, struct mm_struct *mm) +{ + unsigned long flags; + unsigned char *tags = NULL; + + spin_lock_irqsave(&mm->context.tag_lock, flags); + tag_desc->tag_users--; + if (tag_desc->tag_users == 0) { + tag_desc->start = tag_desc->end = 0; + /* Do not free up the tag storage space allocated + * by the first descriptor. This is persistent + * emergency tag storage space for the task. + */ + if (tag_desc != mm->context.tag_store) { + tags = tag_desc->tags; + tag_desc->tags = NULL; + } + } + spin_unlock_irqrestore(&mm->context.tag_lock, flags); + kfree(tags); +} + +#define tag_start(addr, tag_desc) \ + ((tag_desc)->tags + ((addr - (tag_desc)->start)/(2*adi_blksize()))) + +/* Retrieve any saved ADI tags for the page being swapped back in and + * restore these tags to the newly allocated physical page. + */ +void adi_restore_tags(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, pte_t pte) +{ + unsigned char *tag; + tag_storage_desc_t *tag_desc; + unsigned long paddr, tmp, version1, version2; + + /* Check if the swapped out page has an ADI version + * saved. If yes, restore version tag to the newly + * allocated page. + */ + tag_desc = find_tag_store(mm, vma, addr); + if (tag_desc == NULL) + return; + + tag = tag_start(addr, tag_desc); + paddr = pte_val(pte) & _PAGE_PADDR_4V; + for (tmp = paddr; tmp < (paddr+PAGE_SIZE); tmp += adi_blksize()) { + version1 = (*tag) >> 4; + version2 = (*tag) & 0x0f; + *tag++ = 0; + asm volatile("stxa %0, [%1] %2\n\t" + : + : "r" (version1), "r" (tmp), + "i" (ASI_MCD_REAL)); + tmp += adi_blksize(); + asm volatile("stxa %0, [%1] %2\n\t" + : + : "r" (version2), "r" (tmp), + "i" (ASI_MCD_REAL)); + } + asm volatile("membar #Sync\n\t"); + + /* Check and mark this tag space for release later if + * the swapped in page was the last user of tag space + */ + del_tag_store(tag_desc, mm); +} + +/* A page is about to be swapped out. Save any ADI tags associated with + * this physical page so they can be restored later when the page is swapped + * back in. + */ +int adi_save_tags(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, pte_t oldpte) +{ + unsigned char *tag; + tag_storage_desc_t *tag_desc; + unsigned long version1, version2, paddr, tmp; + + tag_desc = alloc_tag_store(mm, vma, addr); + if (tag_desc == NULL) + return -1; + + tag = tag_start(addr, tag_desc); + paddr = pte_val(oldpte) & _PAGE_PADDR_4V; + for (tmp = paddr; tmp < (paddr+PAGE_SIZE); tmp += adi_blksize()) { + asm volatile("ldxa [%1] %2, %0\n\t" + : "=r" (version1) + : "r" (tmp), "i" (ASI_MCD_REAL)); + tmp += adi_blksize(); + asm volatile("ldxa [%1] %2, %0\n\t" + : "=r" (version2) + : "r" (tmp), "i" (ASI_MCD_REAL)); + *tag = (version1 << 4) | version2; + tag++; + } + + return 0; +} diff --git a/arch/sparc/kernel/apc.c b/arch/sparc/kernel/apc.c index eefda32b595e..849db20e7165 100644 --- a/arch/sparc/kernel/apc.c +++ b/arch/sparc/kernel/apc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* apc - Driver implementation for power management functions * of Aurora Personality Chip (APC) on SPARCstation-4/5 and * derivatives. @@ -12,12 +13,12 @@ #include <linux/miscdevice.h> #include <linux/pm.h> #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/platform_device.h> #include <linux/module.h> #include <asm/io.h> #include <asm/oplib.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/auxio.h> #include <asm/apc.h> #include <asm/processor.h> @@ -27,7 +28,6 @@ * #define APC_DEBUG_LED */ -#define APC_MINOR MISC_DYNAMIC_MINOR #define APC_OBPNAME "power-management" #define APC_DEVNAME "apc" @@ -137,7 +137,7 @@ static const struct file_operations apc_fops = { .llseek = noop_llseek, }; -static struct miscdevice apc_miscdev = { APC_MINOR, APC_DEVNAME, &apc_fops }; +static struct miscdevice apc_miscdev = { MISC_DYNAMIC_MINOR, APC_DEVNAME, &apc_fops }; static int apc_probe(struct platform_device *op) { @@ -167,7 +167,7 @@ static int apc_probe(struct platform_device *op) return 0; } -static struct of_device_id apc_match[] = { +static const struct of_device_id apc_match[] = { { .name = APC_OBPNAME, }, @@ -178,7 +178,6 @@ MODULE_DEVICE_TABLE(of, apc_match); static struct platform_driver apc_driver = { .driver = { .name = "apc", - .owner = THIS_MODULE, .of_match_table = apc_match, }, .probe = apc_probe, diff --git a/arch/sparc/kernel/asm-offsets.c b/arch/sparc/kernel/asm-offsets.c index f76389a32342..6e660bde48dd 100644 --- a/arch/sparc/kernel/asm-offsets.c +++ b/arch/sparc/kernel/asm-offsets.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * This program is used to generate definitions needed by * assembly language modules. @@ -9,22 +10,24 @@ * * On sparc, thread_info data is static and TI_XXX offsets are computed by hand. */ +#define COMPILE_OFFSETS #include <linux/sched.h> +#include <linux/mm_types.h> // #include <linux/mm.h> #include <linux/kbuild.h> #include <asm/hibernate.h> #ifdef CONFIG_SPARC32 -int sparc32_foo(void) +static int __used sparc32_foo(void) { DEFINE(AOFF_thread_fork_kpsr, offsetof(struct thread_struct, fork_kpsr)); return 0; } #else -int sparc64_foo(void) +static int __used sparc64_foo(void) { #ifdef CONFIG_HIBERNATION BLANK(); @@ -43,7 +46,7 @@ int sparc64_foo(void) } #endif -int foo(void) +static int __used foo(void) { BLANK(); DEFINE(AOFF_task_thread, offsetof(struct task_struct, thread)); diff --git a/arch/sparc/kernel/audit.c b/arch/sparc/kernel/audit.c index 8fff0ac63d56..b092274eca79 100644 --- a/arch/sparc/kernel/audit.c +++ b/arch/sparc/kernel/audit.c @@ -1,29 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/init.h> #include <linux/types.h> #include <linux/audit.h> #include <asm/unistd.h> -static unsigned dir_class[] = { +#include "kernel.h" + +static unsigned int dir_class[] = { #include <asm-generic/audit_dir_write.h> ~0U }; -static unsigned read_class[] = { +static unsigned int read_class[] = { #include <asm-generic/audit_read.h> ~0U }; -static unsigned write_class[] = { +static unsigned int write_class[] = { #include <asm-generic/audit_write.h> ~0U }; -static unsigned chattr_class[] = { +static unsigned int chattr_class[] = { #include <asm-generic/audit_change_attr.h> ~0U }; -static unsigned signal_class[] = { +static unsigned int signal_class[] = { #include <asm-generic/audit_signal.h> ~0U }; @@ -37,35 +40,31 @@ int audit_classify_arch(int arch) return 0; } -int audit_classify_syscall(int abi, unsigned syscall) +int audit_classify_syscall(int abi, unsigned int syscall) { #ifdef CONFIG_COMPAT - extern int sparc32_classify_syscall(unsigned); if (abi == AUDIT_ARCH_SPARC) return sparc32_classify_syscall(syscall); #endif switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: - return 5; + return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: - return 0; + return AUDITSC_NATIVE; } } static int __init audit_classes_init(void) { #ifdef CONFIG_COMPAT - extern __u32 sparc32_dir_class[]; - extern __u32 sparc32_write_class[]; - extern __u32 sparc32_read_class[]; - extern __u32 sparc32_chattr_class[]; - extern __u32 sparc32_signal_class[]; audit_register_class(AUDIT_CLASS_WRITE_32, sparc32_write_class); audit_register_class(AUDIT_CLASS_READ_32, sparc32_read_class); audit_register_class(AUDIT_CLASS_DIR_WRITE_32, sparc32_dir_class); diff --git a/arch/sparc/kernel/auxio_32.c b/arch/sparc/kernel/auxio_32.c index e20cc55fb768..989860e890c4 100644 --- a/arch/sparc/kernel/auxio_32.c +++ b/arch/sparc/kernel/auxio_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* auxio.c: Probing for the Sparc AUXIO register at boot time. * * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) @@ -7,14 +8,16 @@ #include <linux/init.h> #include <linux/spinlock.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/export.h> + #include <asm/oplib.h> #include <asm/io.h> #include <asm/auxio.h> #include <asm/string.h> /* memset(), Linux has no bzero() */ #include <asm/cpu_type.h> +#include "kernel.h" + /* Probe and map in the Auxiliary I/O register */ /* auxio_register is not static because it is referenced @@ -103,7 +106,7 @@ EXPORT_SYMBOL(set_auxio); /* sun4m power control register (AUXIO2) */ -volatile unsigned char * auxio_power_register = NULL; +volatile u8 __iomem *auxio_power_register = NULL; void __init auxio_power_probe(void) { @@ -127,8 +130,8 @@ void __init auxio_power_probe(void) r.flags = regs.which_io & 0xF; r.start = regs.phys_addr; r.end = regs.phys_addr + regs.reg_size - 1; - auxio_power_register = (unsigned char *) of_ioremap(&r, 0, - regs.reg_size, "auxpower"); + auxio_power_register = + (u8 __iomem *)of_ioremap(&r, 0, regs.reg_size, "auxpower"); /* Display a quick message on the console. */ if (auxio_power_register) diff --git a/arch/sparc/kernel/auxio_64.c b/arch/sparc/kernel/auxio_64.c index 86e55778e4af..2a2800d21325 100644 --- a/arch/sparc/kernel/auxio_64.c +++ b/arch/sparc/kernel/auxio_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* auxio.c: Probing for the Sparc AUXIO register at boot time. * * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) @@ -9,7 +10,8 @@ #include <linux/kernel.h> #include <linux/init.h> #include <linux/ioport.h> -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/platform_device.h> #include <asm/prom.h> #include <asm/io.h> @@ -86,7 +88,6 @@ void auxio_set_lte(int on) __auxio_sbus_set_lte(on); break; case AUXIO_TYPE_EBUS: - /* FALL-THROUGH */ default: break; } @@ -107,23 +108,22 @@ static int auxio_probe(struct platform_device *dev) struct device_node *dp = dev->dev.of_node; unsigned long size; - if (!strcmp(dp->parent->name, "ebus")) { + if (of_node_name_eq(dp->parent, "ebus")) { auxio_devtype = AUXIO_TYPE_EBUS; size = sizeof(u32); - } else if (!strcmp(dp->parent->name, "sbus")) { + } else if (of_node_name_eq(dp->parent, "sbus")) { auxio_devtype = AUXIO_TYPE_SBUS; size = 1; } else { - printk("auxio: Unknown parent bus type [%s]\n", - dp->parent->name); + printk("auxio: Unknown parent bus type [%pOFn]\n", + dp->parent); return -ENODEV; } auxio_register = of_ioremap(&dev->resource[0], 0, size, "auxio"); if (!auxio_register) return -ENODEV; - printk(KERN_INFO "AUXIO: Found device at %s\n", - dp->full_name); + printk(KERN_INFO "AUXIO: Found device at %pOF\n", dp); if (auxio_devtype == AUXIO_TYPE_EBUS) auxio_set_led(AUXIO_LED_ON); @@ -135,7 +135,6 @@ static struct platform_driver auxio_driver = { .probe = auxio_probe, .driver = { .name = "auxio", - .owner = THIS_MODULE, .of_match_table = auxio_match, }, }; diff --git a/arch/sparc/kernel/btext.c b/arch/sparc/kernel/btext.c index 57073e56ba9e..2bf558a0c568 100644 --- a/arch/sparc/kernel/btext.c +++ b/arch/sparc/kernel/btext.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Procedures for drawing on the screen early on in the boot process. * @@ -7,6 +8,7 @@ #include <linux/string.h> #include <linux/init.h> #include <linux/console.h> +#include <linux/font.h> #include <asm/btext.h> #include <asm/oplib.h> @@ -19,11 +21,11 @@ static void scrollscreen(void); #endif static void draw_byte(unsigned char c, long locX, long locY); -static void draw_byte_32(unsigned char *bits, unsigned int *base, int rb); -static void draw_byte_16(unsigned char *bits, unsigned int *base, int rb); -static void draw_byte_8(unsigned char *bits, unsigned int *base, int rb); +static void draw_byte_32(const unsigned char *bits, unsigned int *base, int rb); +static void draw_byte_16(const unsigned char *bits, unsigned int *base, int rb); +static void draw_byte_8(const unsigned char *bits, unsigned int *base, int rb); -#define __force_data __attribute__((__section__(".data"))) +#define __force_data __section(".data") static int g_loc_X __force_data; static int g_loc_Y __force_data; @@ -35,10 +37,6 @@ static int dispDeviceDepth __force_data; static int dispDeviceRect[4] __force_data; static unsigned char *dispDeviceBase __force_data; -#define cmapsz (16*256) - -static unsigned char vga_font[cmapsz]; - static int __init btext_initialize(phandle node) { unsigned int width, height, depth, pitch; @@ -137,7 +135,7 @@ static void scrollscreen(void) } #endif /* ndef NO_SCROLL */ -void btext_drawchar(char c) +static void btext_drawchar(char c) { int cline = 0; #ifdef NO_SCROLL @@ -193,7 +191,8 @@ static void btext_drawtext(const char *c, unsigned int len) static void draw_byte(unsigned char c, long locX, long locY) { unsigned char *base = calc_base(locX << 3, locY << 4); - unsigned char *font = &vga_font[((unsigned int)c) * 16]; + unsigned int font_index = c * 16; + const unsigned char *font = font_sun_8x16.data + font_index; int rb = dispDeviceRowBytes; switch(dispDeviceDepth) { @@ -238,7 +237,7 @@ static unsigned int expand_bits_16[4] = { }; -static void draw_byte_32(unsigned char *font, unsigned int *base, int rb) +static void draw_byte_32(const unsigned char *font, unsigned int *base, int rb) { int l, bits; int fg = 0xFFFFFFFFUL; @@ -259,7 +258,7 @@ static void draw_byte_32(unsigned char *font, unsigned int *base, int rb) } } -static void draw_byte_16(unsigned char *font, unsigned int *base, int rb) +static void draw_byte_16(const unsigned char *font, unsigned int *base, int rb) { int l, bits; int fg = 0xFFFFFFFFUL; @@ -277,7 +276,7 @@ static void draw_byte_16(unsigned char *font, unsigned int *base, int rb) } } -static void draw_byte_8(unsigned char *font, unsigned int *base, int rb) +static void draw_byte_8(const unsigned char *font, unsigned int *base, int rb) { int l, bits; int fg = 0x0F0F0F0FUL; @@ -325,348 +324,3 @@ int __init btext_find_display(void) } return ret; } - -static unsigned char vga_font[cmapsz] = { -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x81, 0xa5, 0x81, 0x81, 0xbd, -0x99, 0x81, 0x81, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0xff, -0xdb, 0xff, 0xff, 0xc3, 0xe7, 0xff, 0xff, 0x7e, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x6c, 0xfe, 0xfe, 0xfe, 0xfe, 0x7c, 0x38, 0x10, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x38, 0x7c, 0xfe, -0x7c, 0x38, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, -0x3c, 0x3c, 0xe7, 0xe7, 0xe7, 0x18, 0x18, 0x3c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x18, 0x3c, 0x7e, 0xff, 0xff, 0x7e, 0x18, 0x18, 0x3c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x3c, -0x3c, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, -0xff, 0xff, 0xe7, 0xc3, 0xc3, 0xe7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x66, 0x42, 0x42, 0x66, 0x3c, 0x00, -0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc3, 0x99, 0xbd, -0xbd, 0x99, 0xc3, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x1e, 0x0e, -0x1a, 0x32, 0x78, 0xcc, 0xcc, 0xcc, 0xcc, 0x78, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x3c, 0x66, 0x66, 0x66, 0x66, 0x3c, 0x18, 0x7e, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x33, 0x3f, 0x30, 0x30, 0x30, -0x30, 0x70, 0xf0, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x63, -0x7f, 0x63, 0x63, 0x63, 0x63, 0x67, 0xe7, 0xe6, 0xc0, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x18, 0x18, 0xdb, 0x3c, 0xe7, 0x3c, 0xdb, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfe, 0xf8, -0xf0, 0xe0, 0xc0, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x06, 0x0e, -0x1e, 0x3e, 0xfe, 0x3e, 0x1e, 0x0e, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x18, 0x3c, 0x7e, 0x18, 0x18, 0x18, 0x7e, 0x3c, 0x18, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, -0x66, 0x00, 0x66, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xdb, -0xdb, 0xdb, 0x7b, 0x1b, 0x1b, 0x1b, 0x1b, 0x1b, 0x00, 0x00, 0x00, 0x00, -0x00, 0x7c, 0xc6, 0x60, 0x38, 0x6c, 0xc6, 0xc6, 0x6c, 0x38, 0x0c, 0xc6, -0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0xfe, 0xfe, 0xfe, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x3c, -0x7e, 0x18, 0x18, 0x18, 0x7e, 0x3c, 0x18, 0x7e, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x18, 0x3c, 0x7e, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x7e, 0x3c, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x18, 0x0c, 0xfe, 0x0c, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x60, 0xfe, 0x60, 0x30, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0xc0, -0xc0, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x24, 0x66, 0xff, 0x66, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x10, 0x38, 0x38, 0x7c, 0x7c, 0xfe, 0xfe, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xfe, 0x7c, 0x7c, -0x38, 0x38, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x18, 0x3c, 0x3c, 0x3c, 0x18, 0x18, 0x18, 0x00, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x66, 0x66, 0x24, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6c, -0x6c, 0xfe, 0x6c, 0x6c, 0x6c, 0xfe, 0x6c, 0x6c, 0x00, 0x00, 0x00, 0x00, -0x18, 0x18, 0x7c, 0xc6, 0xc2, 0xc0, 0x7c, 0x06, 0x06, 0x86, 0xc6, 0x7c, -0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc2, 0xc6, 0x0c, 0x18, -0x30, 0x60, 0xc6, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x6c, -0x6c, 0x38, 0x76, 0xdc, 0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, -0x00, 0x30, 0x30, 0x30, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x18, 0x30, 0x30, 0x30, 0x30, -0x30, 0x30, 0x18, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x18, -0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x18, 0x30, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x3c, 0xff, 0x3c, 0x66, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x7e, -0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x30, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x02, 0x06, 0x0c, 0x18, 0x30, 0x60, 0xc0, 0x80, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x7c, 0xc6, 0xc6, 0xce, 0xde, 0xf6, 0xe6, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x38, 0x78, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, -0x06, 0x0c, 0x18, 0x30, 0x60, 0xc0, 0xc6, 0xfe, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x7c, 0xc6, 0x06, 0x06, 0x3c, 0x06, 0x06, 0x06, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x1c, 0x3c, 0x6c, 0xcc, 0xfe, -0x0c, 0x0c, 0x0c, 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xc0, -0xc0, 0xc0, 0xfc, 0x06, 0x06, 0x06, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x38, 0x60, 0xc0, 0xc0, 0xfc, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xc6, 0x06, 0x06, 0x0c, 0x18, -0x30, 0x30, 0x30, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, -0xc6, 0xc6, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0x7e, 0x06, 0x06, 0x06, 0x0c, 0x78, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x00, 0x00, -0x00, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x18, 0x18, 0x00, 0x00, 0x00, 0x18, 0x18, 0x30, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x06, 0x0c, 0x18, 0x30, 0x60, 0x30, 0x18, 0x0c, 0x06, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, -0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, -0x30, 0x18, 0x0c, 0x06, 0x0c, 0x18, 0x30, 0x60, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x7c, 0xc6, 0xc6, 0x0c, 0x18, 0x18, 0x18, 0x00, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xde, 0xde, -0xde, 0xdc, 0xc0, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x38, -0x6c, 0xc6, 0xc6, 0xfe, 0xc6, 0xc6, 0xc6, 0xc6, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xfc, 0x66, 0x66, 0x66, 0x7c, 0x66, 0x66, 0x66, 0x66, 0xfc, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x66, 0xc2, 0xc0, 0xc0, 0xc0, -0xc0, 0xc2, 0x66, 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x6c, -0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x6c, 0xf8, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xfe, 0x66, 0x62, 0x68, 0x78, 0x68, 0x60, 0x62, 0x66, 0xfe, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0x66, 0x62, 0x68, 0x78, 0x68, -0x60, 0x60, 0x60, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x66, -0xc2, 0xc0, 0xc0, 0xde, 0xc6, 0xc6, 0x66, 0x3a, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xc6, 0xc6, 0xc6, 0xc6, 0xfe, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x0c, -0x0c, 0x0c, 0x0c, 0x0c, 0xcc, 0xcc, 0xcc, 0x78, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xe6, 0x66, 0x66, 0x6c, 0x78, 0x78, 0x6c, 0x66, 0x66, 0xe6, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x60, 0x60, 0x60, 0x60, 0x60, -0x60, 0x62, 0x66, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0xe7, -0xff, 0xff, 0xdb, 0xc3, 0xc3, 0xc3, 0xc3, 0xc3, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xc6, 0xe6, 0xf6, 0xfe, 0xde, 0xce, 0xc6, 0xc6, 0xc6, 0xc6, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, -0xc6, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x66, -0x66, 0x66, 0x7c, 0x60, 0x60, 0x60, 0x60, 0xf0, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xd6, 0xde, 0x7c, -0x0c, 0x0e, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x66, 0x66, 0x66, 0x7c, 0x6c, -0x66, 0x66, 0x66, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, -0xc6, 0x60, 0x38, 0x0c, 0x06, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xff, 0xdb, 0x99, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, -0xc6, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0xc3, -0xc3, 0xc3, 0xc3, 0xc3, 0xc3, 0x66, 0x3c, 0x18, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xc3, 0xc3, 0xc3, 0xc3, 0xc3, 0xdb, 0xdb, 0xff, 0x66, 0x66, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0xc3, 0x66, 0x3c, 0x18, 0x18, -0x3c, 0x66, 0xc3, 0xc3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0xc3, -0xc3, 0x66, 0x3c, 0x18, 0x18, 0x18, 0x18, 0x3c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xff, 0xc3, 0x86, 0x0c, 0x18, 0x30, 0x60, 0xc1, 0xc3, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x30, 0x30, 0x30, 0x30, 0x30, -0x30, 0x30, 0x30, 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, -0xc0, 0xe0, 0x70, 0x38, 0x1c, 0x0e, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x3c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x3c, -0x00, 0x00, 0x00, 0x00, 0x10, 0x38, 0x6c, 0xc6, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, -0x30, 0x30, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x0c, 0x7c, -0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x60, -0x60, 0x78, 0x6c, 0x66, 0x66, 0x66, 0x66, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, 0xc0, 0xc0, 0xc0, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x0c, 0x0c, 0x3c, 0x6c, 0xcc, -0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x7c, 0xc6, 0xfe, 0xc0, 0xc0, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x38, 0x6c, 0x64, 0x60, 0xf0, 0x60, 0x60, 0x60, 0x60, 0xf0, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0xcc, 0xcc, -0xcc, 0xcc, 0xcc, 0x7c, 0x0c, 0xcc, 0x78, 0x00, 0x00, 0x00, 0xe0, 0x60, -0x60, 0x6c, 0x76, 0x66, 0x66, 0x66, 0x66, 0xe6, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x18, 0x18, 0x00, 0x38, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x06, 0x00, 0x0e, 0x06, 0x06, -0x06, 0x06, 0x06, 0x06, 0x66, 0x66, 0x3c, 0x00, 0x00, 0x00, 0xe0, 0x60, -0x60, 0x66, 0x6c, 0x78, 0x78, 0x6c, 0x66, 0xe6, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x38, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe6, 0xff, 0xdb, -0xdb, 0xdb, 0xdb, 0xdb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0xdc, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x66, 0x66, -0x66, 0x66, 0x66, 0x7c, 0x60, 0x60, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x76, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0x7c, 0x0c, 0x0c, 0x1e, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x76, 0x66, 0x60, 0x60, 0x60, 0xf0, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xc6, 0x60, -0x38, 0x0c, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x30, -0x30, 0xfc, 0x30, 0x30, 0x30, 0x30, 0x36, 0x1c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0x76, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0xc3, 0xc3, -0xc3, 0x66, 0x3c, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0xc3, 0xc3, 0xc3, 0xdb, 0xdb, 0xff, 0x66, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0x66, 0x3c, 0x18, 0x3c, 0x66, 0xc3, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0xc6, 0xc6, -0xc6, 0xc6, 0xc6, 0x7e, 0x06, 0x0c, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0xfe, 0xcc, 0x18, 0x30, 0x60, 0xc6, 0xfe, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x0e, 0x18, 0x18, 0x18, 0x70, 0x18, 0x18, 0x18, 0x18, 0x0e, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, 0x00, 0x18, -0x18, 0x18, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x18, -0x18, 0x18, 0x0e, 0x18, 0x18, 0x18, 0x18, 0x70, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x76, 0xdc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x38, 0x6c, 0xc6, -0xc6, 0xc6, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x66, -0xc2, 0xc0, 0xc0, 0xc0, 0xc2, 0x66, 0x3c, 0x0c, 0x06, 0x7c, 0x00, 0x00, -0x00, 0x00, 0xcc, 0x00, 0x00, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0x76, -0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x18, 0x30, 0x00, 0x7c, 0xc6, 0xfe, -0xc0, 0xc0, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x38, 0x6c, -0x00, 0x78, 0x0c, 0x7c, 0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xcc, 0x00, 0x00, 0x78, 0x0c, 0x7c, 0xcc, 0xcc, 0xcc, 0x76, -0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x30, 0x18, 0x00, 0x78, 0x0c, 0x7c, -0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x6c, 0x38, -0x00, 0x78, 0x0c, 0x7c, 0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x3c, 0x66, 0x60, 0x60, 0x66, 0x3c, 0x0c, 0x06, -0x3c, 0x00, 0x00, 0x00, 0x00, 0x10, 0x38, 0x6c, 0x00, 0x7c, 0xc6, 0xfe, -0xc0, 0xc0, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, -0x00, 0x7c, 0xc6, 0xfe, 0xc0, 0xc0, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x60, 0x30, 0x18, 0x00, 0x7c, 0xc6, 0xfe, 0xc0, 0xc0, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x66, 0x00, 0x00, 0x38, 0x18, 0x18, -0x18, 0x18, 0x18, 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x3c, 0x66, -0x00, 0x38, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x60, 0x30, 0x18, 0x00, 0x38, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3c, -0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x10, 0x38, 0x6c, 0xc6, 0xc6, -0xfe, 0xc6, 0xc6, 0xc6, 0x00, 0x00, 0x00, 0x00, 0x38, 0x6c, 0x38, 0x00, -0x38, 0x6c, 0xc6, 0xc6, 0xfe, 0xc6, 0xc6, 0xc6, 0x00, 0x00, 0x00, 0x00, -0x18, 0x30, 0x60, 0x00, 0xfe, 0x66, 0x60, 0x7c, 0x60, 0x60, 0x66, 0xfe, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6e, 0x3b, 0x1b, -0x7e, 0xd8, 0xdc, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x6c, -0xcc, 0xcc, 0xfe, 0xcc, 0xcc, 0xcc, 0xcc, 0xce, 0x00, 0x00, 0x00, 0x00, -0x00, 0x10, 0x38, 0x6c, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, 0x00, 0x7c, 0xc6, 0xc6, -0xc6, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x30, 0x18, -0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x30, 0x78, 0xcc, 0x00, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0x76, -0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0x30, 0x18, 0x00, 0xcc, 0xcc, 0xcc, -0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, -0x00, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x7e, 0x06, 0x0c, 0x78, 0x00, -0x00, 0xc6, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, -0xc6, 0xc6, 0xc6, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x7e, -0xc3, 0xc0, 0xc0, 0xc0, 0xc3, 0x7e, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, -0x00, 0x38, 0x6c, 0x64, 0x60, 0xf0, 0x60, 0x60, 0x60, 0x60, 0xe6, 0xfc, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc3, 0x66, 0x3c, 0x18, 0xff, 0x18, -0xff, 0x18, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x66, 0x66, -0x7c, 0x62, 0x66, 0x6f, 0x66, 0x66, 0x66, 0xf3, 0x00, 0x00, 0x00, 0x00, -0x00, 0x0e, 0x1b, 0x18, 0x18, 0x18, 0x7e, 0x18, 0x18, 0x18, 0x18, 0x18, -0xd8, 0x70, 0x00, 0x00, 0x00, 0x18, 0x30, 0x60, 0x00, 0x78, 0x0c, 0x7c, -0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x18, 0x30, -0x00, 0x38, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x18, 0x30, 0x60, 0x00, 0x7c, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x30, 0x60, 0x00, 0xcc, 0xcc, 0xcc, -0xcc, 0xcc, 0xcc, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0xdc, -0x00, 0xdc, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x00, 0x00, 0x00, 0x00, -0x76, 0xdc, 0x00, 0xc6, 0xe6, 0xf6, 0xfe, 0xde, 0xce, 0xc6, 0xc6, 0xc6, -0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0x6c, 0x6c, 0x3e, 0x00, 0x7e, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x6c, 0x6c, -0x38, 0x00, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x30, 0x30, 0x00, 0x30, 0x30, 0x60, 0xc0, 0xc6, 0xc6, 0x7c, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xc0, -0xc0, 0xc0, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0xfe, 0x06, 0x06, 0x06, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0xc0, 0xc0, 0xc2, 0xc6, 0xcc, 0x18, 0x30, 0x60, 0xce, 0x9b, 0x06, -0x0c, 0x1f, 0x00, 0x00, 0x00, 0xc0, 0xc0, 0xc2, 0xc6, 0xcc, 0x18, 0x30, -0x66, 0xce, 0x96, 0x3e, 0x06, 0x06, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, -0x00, 0x18, 0x18, 0x18, 0x3c, 0x3c, 0x3c, 0x18, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x6c, 0xd8, 0x6c, 0x36, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, 0x6c, 0x36, -0x6c, 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x44, 0x11, 0x44, -0x11, 0x44, 0x11, 0x44, 0x11, 0x44, 0x11, 0x44, 0x11, 0x44, 0x11, 0x44, -0x55, 0xaa, 0x55, 0xaa, 0x55, 0xaa, 0x55, 0xaa, 0x55, 0xaa, 0x55, 0xaa, -0x55, 0xaa, 0x55, 0xaa, 0xdd, 0x77, 0xdd, 0x77, 0xdd, 0x77, 0xdd, 0x77, -0xdd, 0x77, 0xdd, 0x77, 0xdd, 0x77, 0xdd, 0x77, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0xf8, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0xf8, 0x18, 0xf8, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0xf6, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x18, 0xf8, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x36, 0x36, 0x36, 0x36, -0x36, 0xf6, 0x06, 0xf6, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0x06, 0xf6, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x36, 0xf6, 0x06, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0xfe, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, 0x18, 0xf8, 0x18, 0xf8, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0xf8, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x1f, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0xff, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x1f, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0xff, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x1f, 0x18, 0x1f, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x37, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x36, 0x37, 0x30, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x30, 0x37, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0xf7, 0x00, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0xff, 0x00, 0xf7, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x36, 0x37, 0x30, 0x37, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x36, 0x36, 0x36, -0x36, 0xf7, 0x00, 0xf7, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x18, 0x18, 0x18, 0x18, 0x18, 0xff, 0x00, 0xff, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0xff, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0xff, 0x00, 0xff, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x3f, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, -0x18, 0x1f, 0x18, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x18, 0x1f, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, -0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x36, 0x36, 0x36, 0xff, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, -0x18, 0x18, 0x18, 0x18, 0x18, 0xff, 0x18, 0xff, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0xf8, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x1f, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, -0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, -0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0xf0, 0xf0, 0xf0, -0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, -0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, -0x0f, 0x0f, 0x0f, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x76, 0xdc, 0xd8, 0xd8, 0xd8, 0xdc, 0x76, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x78, 0xcc, 0xcc, 0xcc, 0xd8, 0xcc, 0xc6, 0xc6, 0xc6, 0xcc, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfe, 0xc6, 0xc6, 0xc0, 0xc0, 0xc0, -0xc0, 0xc0, 0xc0, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0xfe, 0x6c, 0x6c, 0x6c, 0x6c, 0x6c, 0x6c, 0x6c, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0xfe, 0xc6, 0x60, 0x30, 0x18, 0x30, 0x60, 0xc6, 0xfe, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0xd8, 0xd8, -0xd8, 0xd8, 0xd8, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x66, 0x66, 0x66, 0x66, 0x66, 0x7c, 0x60, 0x60, 0xc0, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x76, 0xdc, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x18, 0x3c, 0x66, 0x66, -0x66, 0x3c, 0x18, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, -0x6c, 0xc6, 0xc6, 0xfe, 0xc6, 0xc6, 0x6c, 0x38, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x38, 0x6c, 0xc6, 0xc6, 0xc6, 0x6c, 0x6c, 0x6c, 0x6c, 0xee, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x30, 0x18, 0x0c, 0x3e, 0x66, -0x66, 0x66, 0x66, 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x7e, 0xdb, 0xdb, 0xdb, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x03, 0x06, 0x7e, 0xdb, 0xdb, 0xf3, 0x7e, 0x60, 0xc0, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x30, 0x60, 0x60, 0x7c, 0x60, -0x60, 0x60, 0x30, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, -0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0xc6, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0xfe, 0x00, 0x00, 0xfe, 0x00, 0x00, 0xfe, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x7e, 0x18, -0x18, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, -0x18, 0x0c, 0x06, 0x0c, 0x18, 0x30, 0x00, 0x7e, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x0c, 0x18, 0x30, 0x60, 0x30, 0x18, 0x0c, 0x00, 0x7e, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x1b, 0x1b, 0x1b, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, -0x18, 0x18, 0x18, 0x18, 0xd8, 0xd8, 0xd8, 0x70, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x00, 0x7e, 0x00, 0x18, 0x18, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0xdc, 0x00, -0x76, 0xdc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x6c, 0x6c, -0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x0c, 0x0c, -0x0c, 0x0c, 0x0c, 0xec, 0x6c, 0x6c, 0x3c, 0x1c, 0x00, 0x00, 0x00, 0x00, -0x00, 0xd8, 0x6c, 0x6c, 0x6c, 0x6c, 0x6c, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0xd8, 0x30, 0x60, 0xc8, 0xf8, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, -}; diff --git a/arch/sparc/kernel/central.c b/arch/sparc/kernel/central.c index 052b5a44318f..a1a6485c9183 100644 --- a/arch/sparc/kernel/central.c +++ b/arch/sparc/kernel/central.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* central.c: Central FHC driver for Sunfire/Starfire/Wildfire. * * Copyright (C) 1997, 1999, 2008 David S. Miller (davem@davemloft.net) @@ -9,7 +10,7 @@ #include <linux/export.h> #include <linux/string.h> #include <linux/init.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/platform_device.h> #include <asm/fhc.h> @@ -54,7 +55,7 @@ static int clock_board_calc_nslots(struct clock_board *p) else return 5; } - /* Fallthrough */ + fallthrough; default: return 4; } @@ -152,7 +153,6 @@ static struct platform_driver clock_board_driver = { .probe = clock_board_probe, .driver = { .name = "clock_board", - .owner = THIS_MODULE, .of_match_table = clock_board_match, }, }; @@ -168,7 +168,7 @@ static int fhc_probe(struct platform_device *op) goto out; } - if (!strcmp(op->dev.of_node->parent->name, "central")) + if (of_node_name_eq(op->dev.of_node->parent, "central")) p->central = true; p->pregs = of_ioremap(&op->resource[0], 0, @@ -257,7 +257,6 @@ static struct platform_driver fhc_driver = { .probe = fhc_probe, .driver = { .name = "fhc", - .owner = THIS_MODULE, .of_match_table = fhc_match, }, }; diff --git a/arch/sparc/kernel/cherrs.S b/arch/sparc/kernel/cherrs.S index 4ee1ad420862..7f3d3d264390 100644 --- a/arch/sparc/kernel/cherrs.S +++ b/arch/sparc/kernel/cherrs.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* These get patched into the trap table at boot time * once we know we have a cheetah processor. */ @@ -214,8 +215,7 @@ do_dcpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */ subcc %g1, %g2, %g1 ! Next cacheline bge,pt %icc, 1b nop - ba,pt %xcc, dcpe_icpe_tl1_common - nop + ba,a,pt %xcc, dcpe_icpe_tl1_common do_dcpe_tl1_fatal: sethi %hi(1f), %g7 @@ -224,8 +224,7 @@ do_dcpe_tl1_fatal: mov 0x2, %o0 call cheetah_plus_parity_error add %sp, PTREGS_OFF, %o1 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size do_dcpe_tl1,.-do_dcpe_tl1 .globl do_icpe_tl1 @@ -259,8 +258,7 @@ do_icpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */ subcc %g1, %g2, %g1 bge,pt %icc, 1b nop - ba,pt %xcc, dcpe_icpe_tl1_common - nop + ba,a,pt %xcc, dcpe_icpe_tl1_common do_icpe_tl1_fatal: sethi %hi(1f), %g7 @@ -269,8 +267,7 @@ do_icpe_tl1_fatal: mov 0x3, %o0 call cheetah_plus_parity_error add %sp, PTREGS_OFF, %o1 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size do_icpe_tl1,.-do_icpe_tl1 .type dcpe_icpe_tl1_common,#function @@ -456,7 +453,7 @@ __cheetah_log_error: cmp %g2, 0x63 be c_cee nop - ba,pt %xcc, c_deferred + ba,a,pt %xcc, c_deferred .size __cheetah_log_error,.-__cheetah_log_error /* Cheetah FECC trap handling, we get here from tl{0,1}_fecc diff --git a/arch/sparc/kernel/chmc.c b/arch/sparc/kernel/chmc.c index dbb210d74e21..d4c74d6b2e1b 100644 --- a/arch/sparc/kernel/chmc.c +++ b/arch/sparc/kernel/chmc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* chmc.c: Driver for UltraSPARC-III memory controller. * * Copyright (C) 2001, 2007, 2008 David S. Miller (davem@davemloft.net) @@ -14,7 +15,8 @@ #include <linux/errno.h> #include <linux/init.h> #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <asm/spitfire.h> #include <asm/chmctrl.h> #include <asm/cpudata.h> @@ -28,7 +30,7 @@ #define PFX DRV_MODULE_NAME ": " #define DRV_MODULE_VERSION "0.2" -MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); +MODULE_AUTHOR("David S. Miller <davem@davemloft.net>"); MODULE_DESCRIPTION("UltraSPARC-III memory controller driver"); MODULE_LICENSE("GPL"); MODULE_VERSION(DRV_MODULE_VERSION); @@ -464,8 +466,8 @@ static int jbusmc_probe(struct platform_device *op) mc_list_add(&p->list); - printk(KERN_INFO PFX "UltraSPARC-IIIi memory controller at %s\n", - op->dev.of_node->full_name); + printk(KERN_INFO PFX "UltraSPARC-IIIi memory controller at %pOF\n", + op->dev.of_node); dev_set_drvdata(&op->dev, p); @@ -747,8 +749,8 @@ static int chmc_probe(struct platform_device *op) mc_list_add(&p->list); - printk(KERN_INFO PFX "UltraSPARC-III memory controller at %s [%s]\n", - dp->full_name, + printk(KERN_INFO PFX "UltraSPARC-III memory controller at %pOF [%s]\n", + dp, (p->layout_size ? "ACTIVE" : "INACTIVE")); dev_set_drvdata(&op->dev, p); @@ -786,7 +788,7 @@ static void jbusmc_destroy(struct platform_device *op, struct jbusmc *p) kfree(p); } -static int us3mc_remove(struct platform_device *op) +static void us3mc_remove(struct platform_device *op) { void *p = dev_get_drvdata(&op->dev); @@ -796,7 +798,6 @@ static int us3mc_remove(struct platform_device *op) else if (mc_type == MC_TYPE_JBUS) jbusmc_destroy(op, p); } - return 0; } static const struct of_device_id us3mc_match[] = { @@ -810,7 +811,6 @@ MODULE_DEVICE_TABLE(of, us3mc_match); static struct platform_driver us3mc_driver = { .driver = { .name = "us3mc", - .owner = THIS_MODULE, .of_match_table = us3mc_match, }, .probe = us3mc_probe, diff --git a/arch/sparc/kernel/compat_audit.c b/arch/sparc/kernel/compat_audit.c index d865575b25bf..f1ea0005a729 100644 --- a/arch/sparc/kernel/compat_audit.c +++ b/arch/sparc/kernel/compat_audit.c @@ -1,43 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 #define __32bit_syscall_numbers__ +#include <linux/audit_arch.h> #include <asm/unistd.h> +#include "kernel.h" -unsigned sparc32_dir_class[] = { +unsigned int sparc32_dir_class[] = { #include <asm-generic/audit_dir_write.h> ~0U }; -unsigned sparc32_chattr_class[] = { +unsigned int sparc32_chattr_class[] = { #include <asm-generic/audit_change_attr.h> ~0U }; -unsigned sparc32_write_class[] = { +unsigned int sparc32_write_class[] = { #include <asm-generic/audit_write.h> ~0U }; -unsigned sparc32_read_class[] = { +unsigned int sparc32_read_class[] = { #include <asm-generic/audit_read.h> ~0U }; -unsigned sparc32_signal_class[] = { +unsigned int sparc32_signal_class[] = { #include <asm-generic/audit_signal.h> ~0U }; -int sparc32_classify_syscall(unsigned syscall) +int sparc32_classify_syscall(unsigned int syscall) { switch(syscall) { case __NR_open: - return 2; + return AUDITSC_OPEN; case __NR_openat: - return 3; + return AUDITSC_OPENAT; case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; case __NR_execve: - return 5; + return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; default: - return 1; + return AUDITSC_COMPAT; } } diff --git a/arch/sparc/kernel/cpu.c b/arch/sparc/kernel/cpu.c index 5c5125895db8..79cd6ccfeac0 100644 --- a/arch/sparc/kernel/cpu.c +++ b/arch/sparc/kernel/cpu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* cpu.c: Dinky routines to look for the kind of Sparc cpu * we are on. * @@ -10,9 +11,9 @@ #include <linux/init.h> #include <linux/smp.h> #include <linux/threads.h> +#include <linux/pgtable.h> #include <asm/spitfire.h> -#include <asm/pgtable.h> #include <asm/oplib.h> #include <asm/setup.h> #include <asm/page.h> @@ -22,6 +23,7 @@ #include <asm/cpudata.h> #include "kernel.h" +#include "entry.h" DEFINE_PER_CPU(cpuinfo_sparc, __cpu_data) = { 0 }; EXPORT_PER_CPU_SYMBOL(__cpu_data); @@ -493,6 +495,30 @@ static void __init sun4v_cpu_probe(void) sparc_pmu_type = "niagara5"; break; + case SUN4V_CHIP_SPARC_M6: + sparc_cpu_type = "SPARC-M6"; + sparc_fpu_type = "SPARC-M6 integrated FPU"; + sparc_pmu_type = "sparc-m6"; + break; + + case SUN4V_CHIP_SPARC_M7: + sparc_cpu_type = "SPARC-M7"; + sparc_fpu_type = "SPARC-M7 integrated FPU"; + sparc_pmu_type = "sparc-m7"; + break; + + case SUN4V_CHIP_SPARC_M8: + sparc_cpu_type = "SPARC-M8"; + sparc_fpu_type = "SPARC-M8 integrated FPU"; + sparc_pmu_type = "sparc-m8"; + break; + + case SUN4V_CHIP_SPARC_SN: + sparc_cpu_type = "SPARC-SN"; + sparc_fpu_type = "SPARC-SN integrated FPU"; + sparc_pmu_type = "sparc-sn"; + break; + case SUN4V_CHIP_SPARC64X: sparc_cpu_type = "SPARC64-X"; sparc_fpu_type = "SPARC64-X integrated FPU"; diff --git a/arch/sparc/kernel/cpumap.c b/arch/sparc/kernel/cpumap.c index e4de74c2c9b0..8fcf2d8c6bd2 100644 --- a/arch/sparc/kernel/cpumap.c +++ b/arch/sparc/kernel/cpumap.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* cpumap.c: used for optimizing CPU assignment * * Copyright (C) 2009 Hong H. Pham <hong.pham@windriver.com> @@ -6,7 +7,6 @@ #include <linux/export.h> #include <linux/slab.h> #include <linux/kernel.h> -#include <linux/init.h> #include <linux/cpumask.h> #include <linux/spinlock.h> #include <asm/cpudata.h> @@ -50,7 +50,7 @@ struct cpuinfo_tree { /* Offsets into nodes[] for each level of the tree */ struct cpuinfo_level level[CPUINFO_LVL_MAX]; - struct cpuinfo_node nodes[0]; + struct cpuinfo_node nodes[] __counted_by(total_nodes); }; @@ -194,8 +194,7 @@ static struct cpuinfo_tree *build_cpuinfo_tree(void) n = enumerate_cpuinfo_nodes(tmp_level); - new_tree = kzalloc(sizeof(struct cpuinfo_tree) + - (sizeof(struct cpuinfo_node) * n), GFP_ATOMIC); + new_tree = kzalloc(struct_size(new_tree, nodes, n), GFP_ATOMIC); if (!new_tree) return NULL; @@ -327,6 +326,11 @@ static int iterate_cpu(struct cpuinfo_tree *t, unsigned int root_index) case SUN4V_CHIP_NIAGARA3: case SUN4V_CHIP_NIAGARA4: case SUN4V_CHIP_NIAGARA5: + case SUN4V_CHIP_SPARC_M6: + case SUN4V_CHIP_SPARC_M7: + case SUN4V_CHIP_SPARC_M8: + case SUN4V_CHIP_SPARC_SN: + case SUN4V_CHIP_SPARC64X: rover_inc_table = niagara_iterate_method; break; default: diff --git a/arch/sparc/kernel/cpumap.h b/arch/sparc/kernel/cpumap.h index e639880ab864..7d5b774862e7 100644 --- a/arch/sparc/kernel/cpumap.h +++ b/arch/sparc/kernel/cpumap.h @@ -1,9 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _CPUMAP_H #define _CPUMAP_H #ifdef CONFIG_SMP -extern void cpu_map_rebuild(void); -extern int map_to_cpu(unsigned int index); +void cpu_map_rebuild(void); +int map_to_cpu(unsigned int index); #define cpu_map_init() cpu_map_rebuild() #else #define cpu_map_init() do {} while (0) diff --git a/arch/sparc/kernel/devices.c b/arch/sparc/kernel/devices.c index 3d465e87f7e2..23b6e50d4ada 100644 --- a/arch/sparc/kernel/devices.c +++ b/arch/sparc/kernel/devices.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* devices.c: Initial scan of the prom device tree for important * Sparc device nodes which we need to find. * @@ -19,8 +20,9 @@ #include <asm/smp.h> #include <asm/cpudata.h> #include <asm/cpu_type.h> +#include <asm/setup.h> -extern void clock_stop_probe(void); /* tadpole.c */ +#include "kernel.h" static char *cpu_mid_prop(void) { @@ -131,11 +133,6 @@ void __init device_scan(void) } #endif /* !CONFIG_SMP */ - { - extern void auxio_probe(void); - extern void auxio_power_probe(void); - auxio_probe(); - auxio_power_probe(); - } - clock_stop_probe(); + auxio_probe(); + auxio_power_probe(); } diff --git a/arch/sparc/kernel/dma.c b/arch/sparc/kernel/dma.c deleted file mode 100644 index b667aa6f28f6..000000000000 --- a/arch/sparc/kernel/dma.c +++ /dev/null @@ -1,12 +0,0 @@ -#include <linux/kernel.h> -#include <linux/dma-mapping.h> -#include <linux/dma-debug.h> - -#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 15) - -static int __init dma_init(void) -{ - dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); - return 0; -} -fs_initcall(dma_init); diff --git a/arch/sparc/kernel/ds.c b/arch/sparc/kernel/ds.c index 11d460f6f9cc..f7fc6f2af2f2 100644 --- a/arch/sparc/kernel/ds.c +++ b/arch/sparc/kernel/ds.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* ds.c: Domain Services driver for Logical Domains * * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> @@ -9,6 +10,7 @@ #include <linux/string.h> #include <linux/slab.h> #include <linux/sched.h> +#include <linux/sched/clock.h> #include <linux/delay.h> #include <linux/mutex.h> #include <linux/kthread.h> @@ -31,7 +33,7 @@ static char version[] = DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; -MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); +MODULE_AUTHOR("David S. Miller <davem@davemloft.net>"); MODULE_DESCRIPTION("Sun LDOM domain services driver"); MODULE_LICENSE("GPL"); MODULE_VERSION(DRV_MODULE_VERSION); @@ -85,7 +87,7 @@ struct ds_reg_req { __u64 handle; __u16 major; __u16 minor; - char svc_id[0]; + char svc_id[]; }; struct ds_reg_ack { @@ -528,10 +530,8 @@ static void dr_cpu_mark(struct ds_data *resp, int cpu, int ncpus, } } -static int __cpuinit dr_cpu_configure(struct ds_info *dp, - struct ds_cap_state *cp, - u64 req_num, - cpumask_t *mask) +static int dr_cpu_configure(struct ds_info *dp, struct ds_cap_state *cp, + u64 req_num, cpumask_t *mask) { struct ds_data *resp; int resp_len, ncpus, cpu; @@ -555,7 +555,7 @@ static int __cpuinit dr_cpu_configure(struct ds_info *dp, printk(KERN_INFO "ds-%llu: Starting cpu %d...\n", dp->id, cpu); - err = cpu_up(cpu); + err = add_cpu(cpu); if (err) { __u32 res = DR_CPU_RES_FAILURE; __u32 stat = DR_CPU_STAT_UNCONFIGURED; @@ -611,7 +611,7 @@ static int dr_cpu_unconfigure(struct ds_info *dp, printk(KERN_INFO "ds-%llu: Shutting down cpu %d...\n", dp->id, cpu); - err = cpu_down(cpu); + err = remove_cpu(cpu); if (err) dr_cpu_mark(resp, cpu, ncpus, DR_CPU_RES_FAILURE, @@ -627,9 +627,8 @@ static int dr_cpu_unconfigure(struct ds_info *dp, return 0; } -static void __cpuinit dr_cpu_data(struct ds_info *dp, - struct ds_cap_state *cp, - void *buf, int len) +static void dr_cpu_data(struct ds_info *dp, struct ds_cap_state *cp, void *buf, + int len) { struct ds_data *data = buf; struct dr_cpu_tag *tag = (struct dr_cpu_tag *) (data + 1); @@ -702,12 +701,12 @@ struct ds_var_hdr { struct ds_var_set_msg { struct ds_var_hdr hdr; - char name_and_value[0]; + char name_and_value[]; }; struct ds_var_delete_msg { struct ds_var_hdr hdr; - char name[0]; + char name[]; }; struct ds_var_resp { @@ -782,14 +781,17 @@ void ldom_set_var(const char *var, const char *value) } pkt; char *base, *p; int msg_len, loops; + size_t var_len, value_len; - if (strlen(var) + strlen(value) + 2 > - sizeof(pkt) - sizeof(pkt.header)) { - printk(KERN_ERR PFX - "contents length: %zu, which more than max: %lu," - "so could not set (%s) variable to (%s).\n", - strlen(var) + strlen(value) + 2, - sizeof(pkt) - sizeof(pkt.header), var, value); + var_len = strlen(var) + 1; + value_len = strlen(value) + 1; + + if (var_len + value_len > sizeof(pkt) - sizeof(pkt.header)) { + pr_err(PFX + "contents length: %zu, which more than max: %lu," + "so could not set (%s) variable to (%s).\n", + var_len + value_len, + sizeof(pkt) - sizeof(pkt.header), var, value); return; } @@ -798,10 +800,10 @@ void ldom_set_var(const char *var, const char *value) pkt.header.data.handle = cp->handle; pkt.header.msg.hdr.type = DS_VAR_SET_REQ; base = p = &pkt.header.msg.name_and_value[0]; - strcpy(p, var); - p += strlen(var) + 1; - strcpy(p, value); - p += strlen(value) + 1; + strscpy(p, var, var_len); + p += var_len; + strscpy(p, value, value_len); + p += value_len; msg_len = (sizeof(struct ds_data) + sizeof(struct ds_var_set_msg) + @@ -852,9 +854,8 @@ void ldom_reboot(const char *boot_command) if (boot_command && strlen(boot_command)) { unsigned long len; - strcpy(full_boot_str, "boot "); - strlcpy(full_boot_str + strlen("boot "), boot_command, - sizeof(full_boot_str + strlen("boot "))); + snprintf(full_boot_str, sizeof(full_boot_str), "boot %s", + boot_command); len = strlen(full_boot_str); if (reboot_data_supported) { @@ -879,7 +880,7 @@ void ldom_power_off(void) static void ds_conn_reset(struct ds_info *dp) { - printk(KERN_ERR "ds-%llu: ds_conn_reset() from %pf\n", + printk(KERN_ERR "ds-%llu: ds_conn_reset() from %ps\n", dp->id, __builtin_return_address(0)); } @@ -912,7 +913,7 @@ static int register_services(struct ds_info *dp) pbuf.req.handle = cp->handle; pbuf.req.major = 1; pbuf.req.minor = 0; - strcpy(pbuf.req.svc_id, cp->service_id); + strscpy(pbuf.id_buf, cp->service_id); err = __ds_send(lp, &pbuf, msg_len); if (err > 0) @@ -991,7 +992,7 @@ struct ds_queue_entry { struct ds_info *dp; int req_len; int __pad; - u64 req[0]; + u64 req[]; }; static void process_ds_work(void) @@ -1204,14 +1205,14 @@ static int ds_probe(struct vio_dev *vdev, const struct vio_device_id *id) ds_cfg.tx_irq = vdev->tx_irq; ds_cfg.rx_irq = vdev->rx_irq; - lp = ldc_alloc(vdev->channel_id, &ds_cfg, dp); + lp = ldc_alloc(vdev->channel_id, &ds_cfg, dp, "DS"); if (IS_ERR(lp)) { err = PTR_ERR(lp); goto out_free_ds_states; } dp->lp = lp; - err = ldc_bind(lp, "DS"); + err = ldc_bind(lp); if (err) goto out_free_ldc; @@ -1238,11 +1239,6 @@ out_err: return err; } -static int ds_remove(struct vio_dev *vdev) -{ - return 0; -} - static const struct vio_device_id ds_match[] = { { .type = "domain-services-port", @@ -1253,7 +1249,6 @@ static const struct vio_device_id ds_match[] = { static struct vio_driver ds_driver = { .id_table = ds_match, .probe = ds_probe, - .remove = ds_remove, .name = "ds", }; diff --git a/arch/sparc/kernel/dtlb_miss.S b/arch/sparc/kernel/dtlb_miss.S index 09a6a15a7105..fb9c788437b0 100644 --- a/arch/sparc/kernel/dtlb_miss.S +++ b/arch/sparc/kernel/dtlb_miss.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* DTLB ** ICACHE line 1: Context 0 check and TSB load */ ldxa [%g0] ASI_DMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer ldxa [%g0] ASI_DMMU, %g6 ! Get TAG TARGET diff --git a/arch/sparc/kernel/dtlb_prot.S b/arch/sparc/kernel/dtlb_prot.S index b2c2c5be281c..9f945771bbd1 100644 --- a/arch/sparc/kernel/dtlb_prot.S +++ b/arch/sparc/kernel/dtlb_prot.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * dtlb_prot.S: DTLB protection trap strategy. * This is included directly into the trap table. @@ -24,13 +25,13 @@ mov TLB_TAG_ACCESS, %g4 ! For reload of vaddr /* PROT ** ICACHE line 2: More real fault processing */ + ldxa [%g4] ASI_DMMU, %g5 ! Put tagaccess in %g5 + srlx %g5, PAGE_SHIFT, %g5 + sllx %g5, PAGE_SHIFT, %g5 ! Clear context ID bits bgu,pn %xcc, winfix_trampoline ! Yes, perform winfixup - ldxa [%g4] ASI_DMMU, %g5 ! Put tagaccess in %g5 - ba,pt %xcc, sparc64_realfault_common ! Nope, normal fault mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4 - nop - nop - nop + ba,pt %xcc, sparc64_realfault_common ! Nope, normal fault + nop nop /* PROT ** ICACHE line 3: Unused... */ diff --git a/arch/sparc/kernel/ebus.c b/arch/sparc/kernel/ebus.c index e306fb08ee5e..264b186478f3 100644 --- a/arch/sparc/kernel/ebus.c +++ b/arch/sparc/kernel/ebus.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* ebus.c: EBUS DMA library code. * * Copyright (C) 1997 Eddie C. Dost (ecd@skynet.be) @@ -7,7 +8,6 @@ #include <linux/export.h> #include <linux/kernel.h> #include <linux/types.h> -#include <linux/init.h> #include <linux/interrupt.h> #include <linux/delay.h> diff --git a/arch/sparc/kernel/entry.S b/arch/sparc/kernel/entry.S index e2a030045089..a3fdee4cd6fa 100644 --- a/arch/sparc/kernel/entry.S +++ b/arch/sparc/kernel/entry.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* arch/sparc/kernel/entry.S: Sparc trap low-level entry points. * * Copyright (C) 1995, 2007 David S. Miller (davem@davemloft.net) @@ -7,8 +8,10 @@ * Copyright (C) 1997 Anton Blanchard (anton@progsoc.uts.edu.au) */ +#include <linux/export.h> #include <linux/linkage.h> #include <linux/errno.h> +#include <linux/pgtable.h> #include <asm/head.h> #include <asm/asi.h> @@ -19,7 +22,6 @@ #include <asm/psr.h> #include <asm/vaddrs.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/winmacro.h> #include <asm/signal.h> #include <asm/obio.h> @@ -799,27 +801,12 @@ SUN_PI_(lda [%l4] ASI_M_MMUREGS, %l5) ! read sfsr last RESTORE_ALL .align 4 - .globl sys_nis_syscall -sys_nis_syscall: - mov %o7, %l5 - add %sp, STACKFRAME_SZ, %o0 ! pt_regs *regs arg - call c_sys_nis_syscall - mov %l5, %o7 - sunos_execv: .globl sunos_execv b sys_execve clr %i2 .align 4 - .globl sys_sparc_pipe -sys_sparc_pipe: - mov %o7, %l5 - add %sp, STACKFRAME_SZ, %o0 ! pt_regs *regs arg - call sparc_pipe - mov %l5, %o7 - - .align 4 .globl sys_sigstack sys_sigstack: mov %o7, %l5 @@ -839,7 +826,7 @@ sys_sigreturn: nop call syscall_trace - nop + mov 1, %o1 1: /* We don't want to muck with user registers like a @@ -882,14 +869,11 @@ flush_patch_two: ld [%curptr + TI_TASK], %o4 rd %psr, %g4 WRITE_PAUSE - mov SIGCHLD, %o0 ! arg0: clone flags rd %wim, %g5 WRITE_PAUSE - mov %fp, %o1 ! arg1: usp std %g4, [%o4 + AOFF_task_thread + AOFF_thread_fork_kpsr] - add %sp, STACKFRAME_SZ, %o2 ! arg2: pt_regs ptr - mov 0, %o3 - call sparc_do_fork + add %sp, STACKFRAME_SZ, %o0 + call sparc_fork mov %l5, %o7 /* Whee, kernel threads! */ @@ -901,19 +885,11 @@ flush_patch_three: ld [%curptr + TI_TASK], %o4 rd %psr, %g4 WRITE_PAUSE - - /* arg0,1: flags,usp -- loaded already */ - cmp %o1, 0x0 ! Is new_usp NULL? rd %wim, %g5 WRITE_PAUSE - be,a 1f - mov %fp, %o1 ! yes, use callers usp - andn %o1, 7, %o1 ! no, align to 8 bytes -1: std %g4, [%o4 + AOFF_task_thread + AOFF_thread_fork_kpsr] - add %sp, STACKFRAME_SZ, %o2 ! arg2: pt_regs ptr - mov 0, %o3 - call sparc_do_fork + add %sp, STACKFRAME_SZ, %o0 + call sparc_clone mov %l5, %o7 /* Whee, real vfork! */ @@ -927,13 +903,9 @@ flush_patch_four: rd %wim, %g5 WRITE_PAUSE std %g4, [%o4 + AOFF_task_thread + AOFF_thread_fork_kpsr] - sethi %hi(0x4000 | 0x0100 | SIGCHLD), %o0 - mov %fp, %o1 - or %o0, %lo(0x4000 | 0x0100 | SIGCHLD), %o0 - sethi %hi(sparc_do_fork), %l1 - mov 0, %o3 - jmpl %l1 + %lo(sparc_do_fork), %g0 - add %sp, STACKFRAME_SZ, %o2 + sethi %hi(sparc_vfork), %l1 + jmpl %l1 + %lo(sparc_vfork), %g0 + add %sp, STACKFRAME_SZ, %o0 .align 4 linux_sparc_ni_syscall: @@ -948,7 +920,24 @@ linux_syscall_trace: cmp %o0, 0 bne 3f mov -ENOSYS, %o0 + + /* Syscall tracing can modify the registers. */ + ld [%sp + STACKFRAME_SZ + PT_G1], %g1 + sethi %hi(sys_call_table), %l7 + ld [%sp + STACKFRAME_SZ + PT_I0], %i0 + or %l7, %lo(sys_call_table), %l7 + ld [%sp + STACKFRAME_SZ + PT_I1], %i1 + ld [%sp + STACKFRAME_SZ + PT_I2], %i2 + ld [%sp + STACKFRAME_SZ + PT_I3], %i3 + ld [%sp + STACKFRAME_SZ + PT_I4], %i4 + ld [%sp + STACKFRAME_SZ + PT_I5], %i5 + cmp %g1, NR_syscalls + bgeu 3f + mov -ENOSYS, %o0 + + sll %g1, 2, %l4 mov %i0, %o0 + ld [%l7 + %l4], %l7 mov %i1, %o1 mov %i2, %o2 mov %i3, %o3 @@ -1005,7 +994,7 @@ do_syscall: andcc %l5, _TIF_SYSCALL_TRACE, %g0 mov %i4, %o4 bne linux_syscall_trace - mov %i0, %l5 + mov %i0, %l6 2: call %l7 mov %i5, %o5 @@ -1014,16 +1003,15 @@ do_syscall: st %o0, [%sp + STACKFRAME_SZ + PT_I0] ret_sys_call: - ld [%curptr + TI_FLAGS], %l6 + ld [%curptr + TI_FLAGS], %l5 cmp %o0, -ERESTART_RESTARTBLOCK ld [%sp + STACKFRAME_SZ + PT_PSR], %g3 set PSR_C, %g2 bgeu 1f - andcc %l6, _TIF_SYSCALL_TRACE, %g0 + andcc %l5, _TIF_SYSCALL_TRACE, %g0 /* System call success, clear Carry condition code. */ andn %g3, %g2, %g3 - clr %l6 st %g3, [%sp + STACKFRAME_SZ + PT_PSR] bne linux_syscall_trace2 ld [%sp + STACKFRAME_SZ + PT_NPC], %l1 /* pc = npc */ @@ -1038,7 +1026,6 @@ ret_sys_call: sub %g0, %o0, %o0 or %g3, %g2, %g3 st %o0, [%sp + STACKFRAME_SZ + PT_I0] - mov 1, %l6 st %g3, [%sp + STACKFRAME_SZ + PT_PSR] bne linux_syscall_trace2 ld [%sp + STACKFRAME_SZ + PT_NPC], %l1 /* pc = npc */ @@ -1190,6 +1177,8 @@ delay_continue: ret restore +EXPORT_SYMBOL(__udelay) +EXPORT_SYMBOL(__ndelay) /* Handle a software breakpoint */ /* We have to inform parent that child has stopped */ @@ -1208,20 +1197,18 @@ breakpoint_trap: RESTORE_ALL #ifdef CONFIG_KGDB - .align 4 - .globl kgdb_trap_low - .type kgdb_trap_low,#function -kgdb_trap_low: + ENTRY(kgdb_trap_low) rd %wim,%l3 SAVE_ALL wr %l0, PSR_ET, %psr WRITE_PAUSE + mov %l7, %o0 ! trap_level call kgdb_trap - add %sp, STACKFRAME_SZ, %o0 + add %sp, STACKFRAME_SZ, %o1 ! struct pt_regs *regs RESTORE_ALL - .size kgdb_trap_low,.-kgdb_trap_low + ENDPROC(kgdb_trap_low) #endif .align 4 @@ -1238,7 +1225,7 @@ flush_patch_exception: kuw_patch1_7win: sll %o3, 6, %o3 /* No matter how much overhead this routine has in the worst - * case scenerio, it is several times better than taking the + * case scenario, it is several times better than taking the * traps with the old method of just doing flush_user_windows(). */ kill_user_windows: diff --git a/arch/sparc/kernel/entry.h b/arch/sparc/kernel/entry.h index cc3c5cb47cda..c746c0fd5d6b 100644 --- a/arch/sparc/kernel/entry.h +++ b/arch/sparc/kernel/entry.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ENTRY_H #define _ENTRY_H @@ -6,40 +7,39 @@ #include <linux/init.h> /* irq */ -extern void handler_irq(int irq, struct pt_regs *regs); +void handler_irq(int irq, struct pt_regs *regs); #ifdef CONFIG_SPARC32 /* traps */ -extern void do_hw_interrupt(struct pt_regs *regs, unsigned long type); -extern void do_illegal_instruction(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); - -extern void do_priv_instruction(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); -extern void do_memaccess_unaligned(struct pt_regs *regs, unsigned long pc, - unsigned long npc, - unsigned long psr); -extern void do_fpd_trap(struct pt_regs *regs, unsigned long pc, +void do_hw_interrupt(struct pt_regs *regs, unsigned long type); +void do_illegal_instruction(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); + +void do_priv_instruction(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void do_memaccess_unaligned(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void do_fpd_trap(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void do_fpe_trap(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void handle_tag_overflow(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void handle_watchpoint(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void handle_reg_access(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void handle_cp_disabled(struct pt_regs *regs, unsigned long pc, unsigned long npc, unsigned long psr); -extern void do_fpe_trap(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); -extern void handle_tag_overflow(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); -extern void handle_watchpoint(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); -extern void handle_reg_access(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); -extern void handle_cp_disabled(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); -extern void handle_cp_exception(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); +void handle_cp_exception(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); /* entry.S */ -extern void fpsave(unsigned long *fpregs, unsigned long *fsr, - void *fpqueue, unsigned long *fpqdepth); -extern void fpload(unsigned long *fpregs, unsigned long *fsr); +void fpsave(unsigned long *fpregs, unsigned long *fsr, + void *fpqueue, unsigned long *fpqdepth); +void fpload(unsigned long *fpregs, unsigned long *fsr); #else /* CONFIG_SPARC32 */ @@ -66,124 +66,121 @@ struct pause_patch_entry { extern struct pause_patch_entry __pause_3insn_patch, __pause_3insn_patch_end; -extern void __init per_cpu_patch(void); -extern void sun4v_patch_1insn_range(struct sun4v_1insn_patch_entry *, - struct sun4v_1insn_patch_entry *); -extern void sun4v_patch_2insn_range(struct sun4v_2insn_patch_entry *, - struct sun4v_2insn_patch_entry *); -extern void __init sun4v_patch(void); -extern void __init boot_cpu_id_too_large(int cpu); +void sun4v_patch_1insn_range(struct sun4v_1insn_patch_entry *, + struct sun4v_1insn_patch_entry *); +void sun4v_patch_2insn_range(struct sun4v_2insn_patch_entry *, + struct sun4v_2insn_patch_entry *); +void sun_m7_patch_2insn_range(struct sun4v_2insn_patch_entry *, + struct sun4v_2insn_patch_entry *); extern unsigned int dcache_parity_tl1_occurred; extern unsigned int icache_parity_tl1_occurred; -extern asmlinkage void sparc_breakpoint(struct pt_regs *regs); -extern void timer_interrupt(int irq, struct pt_regs *regs); - -extern void do_notify_resume(struct pt_regs *regs, - unsigned long orig_i0, - unsigned long thread_info_flags); - -extern asmlinkage int syscall_trace_enter(struct pt_regs *regs); -extern asmlinkage void syscall_trace_leave(struct pt_regs *regs); - -extern void bad_trap_tl1(struct pt_regs *regs, long lvl); - -extern void do_fpe_common(struct pt_regs *regs); -extern void do_fpieee(struct pt_regs *regs); -extern void do_fpother(struct pt_regs *regs); -extern void do_tof(struct pt_regs *regs); -extern void do_div0(struct pt_regs *regs); -extern void do_illegal_instruction(struct pt_regs *regs); -extern void mem_address_unaligned(struct pt_regs *regs, - unsigned long sfar, - unsigned long sfsr); -extern void sun4v_do_mna(struct pt_regs *regs, - unsigned long addr, - unsigned long type_ctx); -extern void do_privop(struct pt_regs *regs); -extern void do_privact(struct pt_regs *regs); -extern void do_cee(struct pt_regs *regs); -extern void do_cee_tl1(struct pt_regs *regs); -extern void do_dae_tl1(struct pt_regs *regs); -extern void do_iae_tl1(struct pt_regs *regs); -extern void do_div0_tl1(struct pt_regs *regs); -extern void do_fpdis_tl1(struct pt_regs *regs); -extern void do_fpieee_tl1(struct pt_regs *regs); -extern void do_fpother_tl1(struct pt_regs *regs); -extern void do_ill_tl1(struct pt_regs *regs); -extern void do_irq_tl1(struct pt_regs *regs); -extern void do_lddfmna_tl1(struct pt_regs *regs); -extern void do_stdfmna_tl1(struct pt_regs *regs); -extern void do_paw(struct pt_regs *regs); -extern void do_paw_tl1(struct pt_regs *regs); -extern void do_vaw(struct pt_regs *regs); -extern void do_vaw_tl1(struct pt_regs *regs); -extern void do_tof_tl1(struct pt_regs *regs); -extern void do_getpsr(struct pt_regs *regs); - -extern void spitfire_insn_access_exception(struct pt_regs *regs, - unsigned long sfsr, - unsigned long sfar); -extern void spitfire_insn_access_exception_tl1(struct pt_regs *regs, - unsigned long sfsr, - unsigned long sfar); -extern void spitfire_data_access_exception(struct pt_regs *regs, - unsigned long sfsr, - unsigned long sfar); -extern void spitfire_data_access_exception_tl1(struct pt_regs *regs, - unsigned long sfsr, - unsigned long sfar); -extern void spitfire_access_error(struct pt_regs *regs, - unsigned long status_encoded, - unsigned long afar); - -extern void cheetah_fecc_handler(struct pt_regs *regs, - unsigned long afsr, - unsigned long afar); -extern void cheetah_cee_handler(struct pt_regs *regs, - unsigned long afsr, - unsigned long afar); -extern void cheetah_deferred_handler(struct pt_regs *regs, - unsigned long afsr, - unsigned long afar); -extern void cheetah_plus_parity_error(int type, struct pt_regs *regs); - -extern void sun4v_insn_access_exception(struct pt_regs *regs, - unsigned long addr, - unsigned long type_ctx); -extern void sun4v_insn_access_exception_tl1(struct pt_regs *regs, - unsigned long addr, - unsigned long type_ctx); -extern void sun4v_data_access_exception(struct pt_regs *regs, - unsigned long addr, - unsigned long type_ctx); -extern void sun4v_data_access_exception_tl1(struct pt_regs *regs, - unsigned long addr, - unsigned long type_ctx); -extern void sun4v_resum_error(struct pt_regs *regs, - unsigned long offset); -extern void sun4v_resum_overflow(struct pt_regs *regs); -extern void sun4v_nonresum_error(struct pt_regs *regs, - unsigned long offset); -extern void sun4v_nonresum_overflow(struct pt_regs *regs); +asmlinkage void sparc_breakpoint(struct pt_regs *regs); +void timer_interrupt(int irq, struct pt_regs *regs); + +void do_notify_resume(struct pt_regs *regs, + unsigned long orig_i0, + unsigned long thread_info_flags); + +asmlinkage int syscall_trace_enter(struct pt_regs *regs); +asmlinkage void syscall_trace_leave(struct pt_regs *regs); + +void bad_trap_tl1(struct pt_regs *regs, long lvl); + +void do_fpieee(struct pt_regs *regs); +void do_fpother(struct pt_regs *regs); +void do_tof(struct pt_regs *regs); +void do_div0(struct pt_regs *regs); +void do_illegal_instruction(struct pt_regs *regs); +void mem_address_unaligned(struct pt_regs *regs, + unsigned long sfar, + unsigned long sfsr); +void sun4v_do_mna(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void do_privop(struct pt_regs *regs); +void do_privact(struct pt_regs *regs); +void do_cee(struct pt_regs *regs); +void do_div0_tl1(struct pt_regs *regs); +void do_fpieee_tl1(struct pt_regs *regs); +void do_fpother_tl1(struct pt_regs *regs); +void do_ill_tl1(struct pt_regs *regs); +void do_irq_tl1(struct pt_regs *regs); +void do_lddfmna_tl1(struct pt_regs *regs); +void do_stdfmna_tl1(struct pt_regs *regs); +void do_paw(struct pt_regs *regs); +void do_paw_tl1(struct pt_regs *regs); +void do_vaw(struct pt_regs *regs); +void do_vaw_tl1(struct pt_regs *regs); +void do_tof_tl1(struct pt_regs *regs); +void do_getpsr(struct pt_regs *regs); + +void spitfire_insn_access_exception(struct pt_regs *regs, + unsigned long sfsr, + unsigned long sfar); +void spitfire_insn_access_exception_tl1(struct pt_regs *regs, + unsigned long sfsr, + unsigned long sfar); +void spitfire_data_access_exception(struct pt_regs *regs, + unsigned long sfsr, + unsigned long sfar); +void spitfire_data_access_exception_tl1(struct pt_regs *regs, + unsigned long sfsr, + unsigned long sfar); +void spitfire_access_error(struct pt_regs *regs, + unsigned long status_encoded, + unsigned long afar); + +void cheetah_fecc_handler(struct pt_regs *regs, + unsigned long afsr, + unsigned long afar); +void cheetah_cee_handler(struct pt_regs *regs, + unsigned long afsr, + unsigned long afar); +void cheetah_deferred_handler(struct pt_regs *regs, + unsigned long afsr, + unsigned long afar); +void cheetah_plus_parity_error(int type, struct pt_regs *regs); + +void sun4v_insn_access_exception(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void sun4v_insn_access_exception_tl1(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void sun4v_data_access_exception(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void sun4v_data_access_exception_tl1(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void sun4v_resum_error(struct pt_regs *regs, + unsigned long offset); +void sun4v_resum_overflow(struct pt_regs *regs); +void sun4v_nonresum_error(struct pt_regs *regs, + unsigned long offset); +void sun4v_nonresum_overflow(struct pt_regs *regs); +void sun4v_mem_corrupt_detect_precise(struct pt_regs *regs, + unsigned long addr, + unsigned long context); extern unsigned long sun4v_err_itlb_vaddr; extern unsigned long sun4v_err_itlb_ctx; extern unsigned long sun4v_err_itlb_pte; extern unsigned long sun4v_err_itlb_error; -extern void sun4v_itlb_error_report(struct pt_regs *regs, int tl); +void sun4v_itlb_error_report(struct pt_regs *regs, int tl); extern unsigned long sun4v_err_dtlb_vaddr; extern unsigned long sun4v_err_dtlb_ctx; extern unsigned long sun4v_err_dtlb_pte; extern unsigned long sun4v_err_dtlb_error; -extern void sun4v_dtlb_error_report(struct pt_regs *regs, int tl); -extern void hypervisor_tlbop_error(unsigned long err, - unsigned long op); -extern void hypervisor_tlbop_error_xcall(unsigned long err, - unsigned long op); +void sun4v_dtlb_error_report(struct pt_regs *regs, int tl); +void hypervisor_tlbop_error(unsigned long err, + unsigned long op); +void hypervisor_tlbop_error_xcall(unsigned long err, + unsigned long op); /* WARNING: The error trap handlers in assembly know the precise * layout of the following structure. @@ -249,8 +246,8 @@ struct ino_bucket { extern struct ino_bucket *ivector_table; extern unsigned long ivector_table_pa; -extern void init_irqwork_curcpu(void); -extern void __cpuinit sun4v_register_mondo_queues(int this_cpu); +void init_irqwork_curcpu(void); +void sun4v_register_mondo_queues(int this_cpu); #endif /* CONFIG_SPARC32 */ #endif /* _ENTRY_H */ diff --git a/arch/sparc/kernel/etrap_32.S b/arch/sparc/kernel/etrap_32.S index e3e80d65e39a..9f243f918619 100644 --- a/arch/sparc/kernel/etrap_32.S +++ b/arch/sparc/kernel/etrap_32.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * etrap.S: Sparc trap window preparation for entry into the * Linux kernel. diff --git a/arch/sparc/kernel/etrap_64.S b/arch/sparc/kernel/etrap_64.S index 1276ca2567ba..08cc41f64725 100644 --- a/arch/sparc/kernel/etrap_64.S +++ b/arch/sparc/kernel/etrap_64.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * etrap.S: Preparing for entry into the kernel on Sparc V9. * @@ -38,7 +39,11 @@ etrap_syscall: TRAP_LOAD_THREAD_REG(%g6, %g1) or %g1, %g3, %g1 bne,pn %xcc, 1f sub %sp, STACKFRAME_SZ+TRACEREG_SZ-STACK_BIAS, %g2 - wrpr %g0, 7, %cleanwin +661: wrpr %g0, 7, %cleanwin + .section .fast_win_ctrl_1insn_patch, "ax" + .word 661b + .word 0x85880000 ! allclean + .previous sethi %hi(TASK_REGOFF), %g2 sethi %hi(TSTATE_PEF), %g3 @@ -88,16 +93,30 @@ etrap_save: save %g2, -STACK_BIAS, %sp bne,pn %xcc, 3f mov PRIMARY_CONTEXT, %l4 - rdpr %canrestore, %g3 +661: rdpr %canrestore, %g3 + .section .fast_win_ctrl_1insn_patch, "ax" + .word 661b + nop + .previous + rdpr %wstate, %g2 - wrpr %g0, 0, %canrestore +661: wrpr %g0, 0, %canrestore + .section .fast_win_ctrl_1insn_patch, "ax" + .word 661b + nop + .previous sll %g2, 3, %g2 /* Set TI_SYS_FPDEPTH to 1 and clear TI_SYS_NOERROR. */ mov 1, %l5 sth %l5, [%l6 + TI_SYS_NOERROR] - wrpr %g3, 0, %otherwin +661: wrpr %g3, 0, %otherwin + .section .fast_win_ctrl_1insn_patch, "ax" + .word 661b + .word 0x87880000 ! otherw + .previous + wrpr %g2, 0, %wstate sethi %hi(sparc64_kern_pri_context), %g2 ldx [%g2 + %lo(sparc64_kern_pri_context)], %g3 @@ -132,7 +151,32 @@ etrap_save: save %g2, -STACK_BIAS, %sp stx %g6, [%sp + PTREGS_OFF + PT_V9_G6] stx %g7, [%sp + PTREGS_OFF + PT_V9_G7] or %l7, %l0, %l7 - sethi %hi(TSTATE_TSO | TSTATE_PEF), %l0 +661: sethi %hi(TSTATE_TSO | TSTATE_PEF), %l0 + /* If userspace is using ADI, it could potentially pass + * a pointer with version tag embedded in it. To maintain + * the ADI security, we must enable PSTATE.mcde. Userspace + * would have already set TTE.mcd in an earlier call to + * kernel and set the version tag for the address being + * dereferenced. Setting PSTATE.mcde would ensure any + * access to userspace data through a system call honors + * ADI and does not allow a rogue app to bypass ADI by + * using system calls. Setting PSTATE.mcde only affects + * accesses to virtual addresses that have TTE.mcd set. + * Set PMCDPER to ensure any exceptions caused by ADI + * version tag mismatch are exposed before system call + * returns to userspace. Setting PMCDPER affects only + * writes to virtual addresses that have TTE.mcd set and + * have a version tag set as well. + */ + .section .sun_m7_1insn_patch, "ax" + .word 661b + sethi %hi(TSTATE_TSO | TSTATE_PEF | TSTATE_MCDE), %l0 + .previous +661: nop + .section .sun_m7_1insn_patch, "ax" + .word 661b + .word 0xaf902001 /* wrpr %g0, 1, %pmcdper */ + .previous or %l7, %l0, %l7 wrpr %l2, %tnpc wrpr %l7, (TSTATE_PRIV | TSTATE_IE), %tstate diff --git a/arch/sparc/kernel/fpu_traps.S b/arch/sparc/kernel/fpu_traps.S index a6864826a4bd..051659e29c7a 100644 --- a/arch/sparc/kernel/fpu_traps.S +++ b/arch/sparc/kernel/fpu_traps.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* This is trivial with the new code... */ .globl do_fpdis .type do_fpdis,#function @@ -100,8 +101,8 @@ do_fpdis: fmuld %f0, %f2, %f26 faddd %f0, %f2, %f28 fmuld %f0, %f2, %f30 - b,pt %xcc, fpdis_exit - nop + ba,a,pt %xcc, fpdis_exit + 2: andcc %g5, FPRS_DU, %g0 bne,pt %icc, 3f fzero %f32 @@ -144,8 +145,8 @@ do_fpdis: fmuld %f32, %f34, %f58 faddd %f32, %f34, %f60 fmuld %f32, %f34, %f62 - ba,pt %xcc, fpdis_exit - nop + ba,a,pt %xcc, fpdis_exit + 3: mov SECONDARY_CONTEXT, %g3 add %g6, TI_FPREGS, %g1 @@ -197,8 +198,7 @@ fpdis_exit2: fp_other_bounce: call do_fpother add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size fp_other_bounce,.-fp_other_bounce .align 32 diff --git a/arch/sparc/kernel/ftrace.c b/arch/sparc/kernel/ftrace.c index 03ab022e51c5..eaead3da8e03 100644 --- a/arch/sparc/kernel/ftrace.c +++ b/arch/sparc/kernel/ftrace.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/spinlock.h> #include <linux/hardirq.h> #include <linux/ftrace.h> @@ -81,15 +82,6 @@ int ftrace_update_ftrace_func(ftrace_func_t func) new = ftrace_call_replace(ip, (unsigned long)func); return ftrace_modify_code(ip, old, new); } - -int __init ftrace_dyn_arch_init(void *data) -{ - unsigned long *p = data; - - *p = 0; - - return 0; -} #endif #ifdef CONFIG_FUNCTION_GRAPH_TRACER @@ -129,22 +121,12 @@ unsigned long prepare_ftrace_return(unsigned long parent, unsigned long frame_pointer) { unsigned long return_hooker = (unsigned long) &return_to_handler; - struct ftrace_graph_ent trace; if (unlikely(atomic_read(¤t->tracing_graph_pause))) return parent + 8UL; - if (ftrace_push_return_trace(parent, self_addr, &trace.depth, - frame_pointer) == -EBUSY) - return parent + 8UL; - - trace.func = self_addr; - - /* Only trace if the calling function expects to */ - if (!ftrace_graph_entry(&trace)) { - current->curr_ret_stack--; + if (function_graph_enter(parent, self_addr, frame_pointer, NULL)) return parent + 8UL; - } return return_hooker; } diff --git a/arch/sparc/kernel/getsetcc.S b/arch/sparc/kernel/getsetcc.S index a14d272d2061..181e09fd1c55 100644 --- a/arch/sparc/kernel/getsetcc.S +++ b/arch/sparc/kernel/getsetcc.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ .globl getcc .type getcc,#function getcc: diff --git a/arch/sparc/kernel/head_32.S b/arch/sparc/kernel/head_32.S index 3d92c0a8f6c4..38345460d542 100644 --- a/arch/sparc/kernel/head_32.S +++ b/arch/sparc/kernel/head_32.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * head.S: The initial boot code for the Sparc port of Linux. * @@ -10,6 +11,7 @@ * CompactPCI platform by Eric Brower, 1999. */ +#include <linux/export.h> #include <linux/version.h> #include <linux/init.h> @@ -23,7 +25,7 @@ #include <asm/winmacro.h> #include <asm/thread_info.h> /* TI_UWINMASK */ #include <asm/errno.h> -#include <asm/pgtsrmmu.h> /* SRMMU_PGDIR_SHIFT */ +#include <asm/pgtable.h> /* PGDIR_SHIFT */ .data /* The following are used with the prom_vector node-ops to figure out @@ -60,6 +62,7 @@ sun4e_notsup: */ .globl empty_zero_page empty_zero_page: .skip PAGE_SIZE +EXPORT_SYMBOL(empty_zero_page) .global root_flags .global ram_flags @@ -115,9 +118,12 @@ current_pc: mov %o7, %g3 tst %o0 - be no_sun4u_here + bne 2f mov %g4, %o7 /* Previous %o7. */ - + sethi %hi(no_sun4u_here), %l1 + jmpl %l1 + %lo(no_sun4u_here), %g0 + nop +2: mov %o0, %l0 ! stash away romvec mov %o0, %g7 ! put it here too mov %o1, %l1 ! stash away debug_vec too @@ -192,7 +198,8 @@ halt_notsup: sub %o0, %l6, %o0 call %o1 nop - ba halt_me + sethi %hi(halt_me), %o0 + jmpl %o0 + %lo(halt_me), %g0 nop not_a_sun4: @@ -270,7 +277,7 @@ not_a_sun4: lda [%o1] ASI_M_BYPASS, %o2 ! This is the 0x0 16MB pgd /* Calculate to KERNBASE entry. */ - add %o1, KERNBASE >> (SRMMU_PGDIR_SHIFT - 2), %o3 + add %o1, KERNBASE >> (PGDIR_SHIFT - 2), %o3 /* Poke the entry into the calculated address. */ sta %o2, [%o3] ASI_M_BYPASS @@ -314,7 +321,7 @@ srmmu_not_viking: sll %g1, 0x8, %g1 ! make phys addr for l1 tbl lda [%g1] ASI_M_BYPASS, %g2 ! get level1 entry for 0x0 - add %g1, KERNBASE >> (SRMMU_PGDIR_SHIFT - 2), %g3 + add %g1, KERNBASE >> (PGDIR_SHIFT - 2), %g3 sta %g2, [%g3] ASI_M_BYPASS ! place at KERNBASE entry b go_to_highmem nop ! wheee.... @@ -338,7 +345,7 @@ leon_remap: sll %g1, 0x8, %g1 ! make phys addr for l1 tbl lda [%g1] ASI_M_BYPASS, %g2 ! get level1 entry for 0x0 - add %g1, KERNBASE >> (SRMMU_PGDIR_SHIFT - 2), %g3 + add %g1, KERNBASE >> (PGDIR_SHIFT - 2), %g3 sta %g2, [%g3] ASI_M_BYPASS ! place at KERNBASE entry b go_to_highmem nop ! wheee.... @@ -428,8 +435,11 @@ leon_init: #ifdef CONFIG_SMP ldub [%g2 + %lo(boot_cpu_id)], %g1 cmp %g1, 0xff ! unset means first CPU - bne leon_smp_cpu_startup ! continue only with master + be 1f + sethi %hi(leon_smp_cpu_startup), %g1 + jmpl %g1 + %lo(leon_smp_cpu_startup), %g0 nop +1: #endif /* Get CPU-ID from most significant 4-bit of ASR17 */ rd %asr17, %g1 @@ -512,7 +522,7 @@ continue_boot: /* I want a kernel stack NOW! */ set init_thread_union, %g1 - set (THREAD_SIZE - STACKFRAME_SZ), %g2 + set (THREAD_SIZE - STACKFRAME_SZ - TRACEREG_SZ), %g2 add %g1, %g2, %sp mov 0, %fp /* And for good luck */ @@ -807,9 +817,3 @@ lvl14_save: .word 0 .word 0 .word t_irq14 - - .section ".fixup",#alloc,#execinstr - .globl __ret_efault -__ret_efault: - ret - restore %g0, -EFAULT, %o0 diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S index 26b706a1867d..cf0549134234 100644 --- a/arch/sparc/kernel/head_64.S +++ b/arch/sparc/kernel/head_64.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* head.S: Initial boot code for the Sparc64 port of Linux. * * Copyright (C) 1996, 1997, 2007 David S. Miller (davem@davemloft.net) @@ -8,16 +9,17 @@ #include <linux/version.h> #include <linux/errno.h> +#include <linux/export.h> #include <linux/threads.h> #include <linux/init.h> #include <linux/linkage.h> +#include <linux/pgtable.h> #include <asm/thread_info.h> #include <asm/asi.h> #include <asm/pstate.h> #include <asm/ptrace.h> #include <asm/spitfire.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/errno.h> #include <asm/signal.h> #include <asm/processor.h> @@ -32,7 +34,7 @@ #include <asm/estate.h> #include <asm/sfafsr.h> #include <asm/unistd.h> - + /* This section from from _start to sparc64_boot_end should fit into * 0x0000000000404000 to 0x0000000000408000. */ @@ -95,6 +97,7 @@ sparc64_boot: andn %g1, PSTATE_AM, %g1 wrpr %g1, 0x0, %pstate ba,a,pt %xcc, 1f + nop .globl prom_finddev_name, prom_chosen_path, prom_root_node .globl prom_getprop_name, prom_mmu_name, prom_peer_name @@ -143,6 +146,7 @@ prom_cpu_compatible: .skip 64 prom_root_node: .word 0 +EXPORT_SYMBOL(prom_root_node) prom_mmu_ihandle_cache: .word 0 prom_boot_mapped_pc: @@ -158,6 +162,7 @@ is_sun4v: .word 0 sun4v_chip_type: .word SUN4V_CHIP_INVALID +EXPORT_SYMBOL(sun4v_chip_type) 1: rd %pc, %l0 @@ -282,8 +287,8 @@ sun4v_chip_type: stx %l2, [%l4 + 0x0] ldx [%sp + 2047 + 128 + 0x50], %l3 ! physaddr low /* 4MB align */ - srlx %l3, 22, %l3 - sllx %l3, 22, %l3 + srlx %l3, ILOG2_4MB, %l3 + sllx %l3, ILOG2_4MB, %l3 stx %l3, [%l4 + 0x8] /* Leave service as-is, "call-method" */ @@ -414,29 +419,43 @@ sun4v_chip_type: cmp %g2, 'T' be,pt %xcc, 70f cmp %g2, 'M' + be,pt %xcc, 70f + cmp %g2, 'S' bne,pn %xcc, 49f nop 70: ldub [%g1 + 7], %g2 - cmp %g2, '3' + cmp %g2, CPU_ID_NIAGARA3 be,pt %xcc, 5f mov SUN4V_CHIP_NIAGARA3, %g4 - cmp %g2, '4' + cmp %g2, CPU_ID_NIAGARA4 be,pt %xcc, 5f mov SUN4V_CHIP_NIAGARA4, %g4 - cmp %g2, '5' + cmp %g2, CPU_ID_NIAGARA5 be,pt %xcc, 5f mov SUN4V_CHIP_NIAGARA5, %g4 + cmp %g2, CPU_ID_M6 + be,pt %xcc, 5f + mov SUN4V_CHIP_SPARC_M6, %g4 + cmp %g2, CPU_ID_M7 + be,pt %xcc, 5f + mov SUN4V_CHIP_SPARC_M7, %g4 + cmp %g2, CPU_ID_M8 + be,pt %xcc, 5f + mov SUN4V_CHIP_SPARC_M8, %g4 + cmp %g2, CPU_ID_SONOMA1 + be,pt %xcc, 5f + mov SUN4V_CHIP_SPARC_SN, %g4 ba,pt %xcc, 49f nop 91: sethi %hi(prom_cpu_compatible), %g1 or %g1, %lo(prom_cpu_compatible), %g1 ldub [%g1 + 17], %g2 - cmp %g2, '1' + cmp %g2, CPU_ID_NIAGARA1 be,pt %xcc, 5f mov SUN4V_CHIP_NIAGARA1, %g4 - cmp %g2, '2' + cmp %g2, CPU_ID_NIAGARA2 be,pt %xcc, 5f mov SUN4V_CHIP_NIAGARA2, %g4 @@ -455,9 +474,8 @@ sun4v_chip_type: subcc %g3, 1, %g3 bne,pt %xcc, 41b add %g1, 1, %g1 - mov SUN4V_CHIP_SPARC64X, %g4 ba,pt %xcc, 5f - nop + mov SUN4V_CHIP_SPARC64X, %g4 49: mov SUN4V_CHIP_UNKNOWN, %g4 @@ -542,8 +560,7 @@ sun4u_init: stxa %g0, [%g7] ASI_DMMU membar #Sync - ba,pt %xcc, sun4u_continue - nop + ba,a,pt %xcc, sun4u_continue sun4v_init: /* Set ctx 0 */ @@ -554,14 +571,12 @@ sun4v_init: mov SECONDARY_CONTEXT, %g7 stxa %g0, [%g7] ASI_MMU membar #Sync - ba,pt %xcc, niagara_tlb_fixup - nop + ba,a,pt %xcc, niagara_tlb_fixup sun4u_continue: BRANCH_IF_ANY_CHEETAH(g1, g7, cheetah_tlb_fixup) - ba,pt %xcc, spitfire_tlb_fixup - nop + ba,a,pt %xcc, spitfire_tlb_fixup niagara_tlb_fixup: mov 3, %g2 /* Set TLB type to hypervisor. */ @@ -585,6 +600,18 @@ niagara_tlb_fixup: cmp %g1, SUN4V_CHIP_NIAGARA5 be,pt %xcc, niagara4_patch nop + cmp %g1, SUN4V_CHIP_SPARC_M6 + be,pt %xcc, niagara4_patch + nop + cmp %g1, SUN4V_CHIP_SPARC_M7 + be,pt %xcc, sparc_m7_patch + nop + cmp %g1, SUN4V_CHIP_SPARC_M8 + be,pt %xcc, sparc_m7_patch + nop + cmp %g1, SUN4V_CHIP_SPARC_SN + be,pt %xcc, niagara4_patch + nop call generic_patch_copyops nop @@ -594,6 +621,19 @@ niagara_tlb_fixup: nop ba,a,pt %xcc, 80f + nop + +sparc_m7_patch: + call m7_patch_copyops + nop + call m7_patch_bzero + nop + call m7_patch_pageops + nop + + ba,a,pt %xcc, 80f + nop + niagara4_patch: call niagara4_patch_copyops nop @@ -601,8 +641,11 @@ niagara4_patch: nop call niagara4_patch_pageops nop + call niagara4_patch_fls + nop ba,a,pt %xcc, 80f + nop niagara2_patch: call niagara2_patch_copyops @@ -613,6 +656,7 @@ niagara2_patch: nop ba,a,pt %xcc, 80f + nop niagara_patch: call niagara_patch_copyops @@ -627,8 +671,7 @@ niagara_patch: call hypervisor_patch_cachetlbops nop - ba,pt %xcc, tlb_fixup_done - nop + ba,a,pt %xcc, tlb_fixup_done cheetah_tlb_fixup: mov 2, %g2 /* Set TLB type to cheetah+. */ @@ -647,8 +690,7 @@ cheetah_tlb_fixup: call cheetah_patch_cachetlbops nop - ba,pt %xcc, tlb_fixup_done - nop + ba,a,pt %xcc, tlb_fixup_done spitfire_tlb_fixup: /* Set TLB type to spitfire. */ @@ -660,14 +702,12 @@ tlb_fixup_done: sethi %hi(init_thread_union), %g6 or %g6, %lo(init_thread_union), %g6 ldx [%g6 + TI_TASK], %g4 - mov %sp, %l6 wr %g0, ASI_P, %asi mov 1, %g1 sllx %g1, THREAD_SHIFT, %g1 - sub %g1, (STACKFRAME_SZ + STACK_BIAS), %g1 + sub %g1, (STACKFRAME_SZ + STACK_BIAS + TRACEREG_SZ), %g1 add %g6, %g1, %sp - mov 0, %fp /* Set per-cpu pointer initially to zero, this makes * the boot-cpu use the in-kernel-image per-cpu areas @@ -686,52 +726,14 @@ tlb_fixup_done: call __bzero sub %o1, %o0, %o1 -#ifdef CONFIG_LOCKDEP - /* We have this call this super early, as even prom_init can grab - * spinlocks and thus call into the lockdep code. - */ - call lockdep_init - nop -#endif - - mov %l6, %o1 ! OpenPROM stack call prom_init mov %l7, %o0 ! OpenPROM cif handler - /* Initialize current_thread_info()->cpu as early as possible. - * In order to do that accurately we have to patch up the get_cpuid() - * assembler sequences. And that, in turn, requires that we know - * if we are on a Starfire box or not. While we're here, patch up - * the sun4v sequences as well. + /* To create a one-register-window buffer between the kernel's + * initial stack and the last stack frame we use from the firmware, + * do the rest of the boot from a C helper function. */ - call check_if_starfire - nop - call per_cpu_patch - nop - call sun4v_patch - nop - -#ifdef CONFIG_SMP - call hard_smp_processor_id - nop - cmp %o0, NR_CPUS - blu,pt %xcc, 1f - nop - call boot_cpu_id_too_large - nop - /* Not reached... */ - -1: -#else - mov 0, %o0 -#endif - sth %o0, [%g6 + TI_CPU] - - call prom_init_report - nop - - /* Off we go.... */ - call start_kernel + call start_early_boot nop /* Not reached... */ @@ -802,8 +804,7 @@ setup_trap_table: call %o1 add %sp, (2047 + 128), %o0 - ba,pt %xcc, 2f - nop + ba,a,pt %xcc, 2f 1: sethi %hi(sparc64_ttable_tl0), %o0 set prom_set_trap_table_name, %g2 @@ -842,8 +843,7 @@ setup_trap_table: BRANCH_IF_ANY_CHEETAH(o2, o3, 1f) - ba,pt %xcc, 2f - nop + ba,a,pt %xcc, 2f /* Disable STICK_INT interrupts. */ 1: @@ -896,8 +896,8 @@ sparc64_boot_end: #include "misctrap.S" #include "syscalls.S" #include "helpers.S" -#include "hvcalls.S" #include "sun4v_tlb_miss.S" +#include "sun4v_mcd.S" #include "sun4v_ivec.S" #include "ktlb.S" #include "tsb.S" @@ -941,6 +941,7 @@ swapper_4m_tsb: ! 0x0000000000428000 +#include "hvcalls.S" #include "systbls_64.S" .data @@ -948,37 +949,21 @@ swapper_4m_tsb: .globl prom_tba, tlb_type prom_tba: .xword 0 tlb_type: .word 0 /* Must NOT end up in BSS */ +EXPORT_SYMBOL(tlb_type) .section ".fixup",#alloc,#execinstr - .globl __ret_efault, __retl_efault, __ret_one, __retl_one -ENTRY(__ret_efault) - ret - restore %g0, -EFAULT, %o0 -ENDPROC(__ret_efault) - ENTRY(__retl_efault) retl mov -EFAULT, %o0 ENDPROC(__retl_efault) -ENTRY(__retl_one) - retl - mov 1, %o0 -ENDPROC(__retl_one) - -ENTRY(__ret_one_asi) - wr %g0, ASI_AIUS, %asi - ret - restore %g0, 1, %o0 -ENDPROC(__ret_one_asi) - -ENTRY(__retl_one_asi) - wr %g0, ASI_AIUS, %asi - retl - mov 1, %o0 -ENDPROC(__retl_one_asi) - ENTRY(__retl_o1) retl mov %o1, %o0 ENDPROC(__retl_o1) + +ENTRY(__retl_o1_asi) + wr %o5, 0x0, %asi + retl + mov %o1, %o0 +ENDPROC(__retl_o1_asi) diff --git a/arch/sparc/kernel/helpers.S b/arch/sparc/kernel/helpers.S index 314dd0c9fc5b..9b3f74706cfb 100644 --- a/arch/sparc/kernel/helpers.S +++ b/arch/sparc/kernel/helpers.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ .align 32 .globl __flushw_user .type __flushw_user,#function @@ -15,6 +16,7 @@ __flushw_user: 2: retl nop .size __flushw_user,.-__flushw_user +EXPORT_SYMBOL(__flushw_user) /* Flush %fp and %i7 to the stack for all register * windows active inside of the cpu. This allows @@ -61,3 +63,4 @@ real_hard_smp_processor_id: .size hard_smp_processor_id,.-hard_smp_processor_id #endif .size real_hard_smp_processor_id,.-real_hard_smp_processor_id +EXPORT_SYMBOL_GPL(real_hard_smp_processor_id) diff --git a/arch/sparc/kernel/hvapi.c b/arch/sparc/kernel/hvapi.c index c0a2de0fd624..717ec7ef07f9 100644 --- a/arch/sparc/kernel/hvapi.c +++ b/arch/sparc/kernel/hvapi.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* hvapi.c: Hypervisor API management. * * Copyright (C) 2007 David S. Miller <davem@davemloft.net> @@ -39,6 +40,8 @@ static struct api_info api_table[] = { { .group = HV_GRP_SDIO, }, { .group = HV_GRP_SDIO_ERR, }, { .group = HV_GRP_REBOOT_DATA, }, + { .group = HV_GRP_ATU, .flags = FLAG_PRE_API }, + { .group = HV_GRP_DAX, }, { .group = HV_GRP_NIAG_PERF, .flags = FLAG_PRE_API }, { .group = HV_GRP_FIRE_PERF, }, { .group = HV_GRP_N2_CPU, }, @@ -46,7 +49,9 @@ static struct api_info api_table[] = { { .group = HV_GRP_VF_CPU, }, { .group = HV_GRP_KT_CPU, }, { .group = HV_GRP_VT_CPU, }, + { .group = HV_GRP_T5_CPU, }, { .group = HV_GRP_DIAG, .flags = FLAG_PRE_API }, + { .group = HV_GRP_M7_PERF, }, }; static DEFINE_SPINLOCK(hvapi_lock); @@ -186,7 +191,7 @@ void __init sun4v_hvapi_init(void) group = HV_GRP_CORE; major = 1; - minor = 1; + minor = 6; if (sun4v_hvapi_register(group, major, &minor)) goto bad; diff --git a/arch/sparc/kernel/hvcalls.S b/arch/sparc/kernel/hvcalls.S index f3ab509b76a8..2f865a464576 100644 --- a/arch/sparc/kernel/hvcalls.S +++ b/arch/sparc/kernel/hvcalls.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* %o0: devhandle * %o1: devino * @@ -106,6 +107,17 @@ ENTRY(sun4v_cpu_yield) nop ENDPROC(sun4v_cpu_yield) + /* %o0: cpuid + * + * returns %o0: status + */ +ENTRY(sun4v_cpu_poke) + mov HV_FAST_CPU_POKE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_cpu_poke) + /* %o0: type * %o1: queue paddr * %o2: num queue entries @@ -338,10 +350,12 @@ ENTRY(sun4v_mach_set_watchdog) mov %o1, %o4 mov HV_FAST_MACH_SET_WATCHDOG, %o5 ta HV_FAST_TRAP + brnz,a,pn %o4, 0f stx %o1, [%o4] - retl +0: retl nop ENDPROC(sun4v_mach_set_watchdog) +EXPORT_SYMBOL(sun4v_mach_set_watchdog) /* No inputs and does not return. */ ENTRY(sun4v_mach_sir) @@ -775,6 +789,7 @@ ENTRY(sun4v_niagara_getperf) retl nop ENDPROC(sun4v_niagara_getperf) +EXPORT_SYMBOL(sun4v_niagara_getperf) ENTRY(sun4v_niagara_setperf) mov HV_FAST_SET_PERFREG, %o5 @@ -782,6 +797,7 @@ ENTRY(sun4v_niagara_setperf) retl nop ENDPROC(sun4v_niagara_setperf) +EXPORT_SYMBOL(sun4v_niagara_setperf) ENTRY(sun4v_niagara2_getperf) mov %o0, %o4 @@ -791,6 +807,7 @@ ENTRY(sun4v_niagara2_getperf) retl nop ENDPROC(sun4v_niagara2_getperf) +EXPORT_SYMBOL(sun4v_niagara2_getperf) ENTRY(sun4v_niagara2_setperf) mov HV_FAST_N2_SET_PERFREG, %o5 @@ -798,6 +815,7 @@ ENTRY(sun4v_niagara2_setperf) retl nop ENDPROC(sun4v_niagara2_setperf) +EXPORT_SYMBOL(sun4v_niagara2_setperf) ENTRY(sun4v_reboot_data_set) mov HV_FAST_REBOOT_DATA_SET, %o5 @@ -821,3 +839,92 @@ ENTRY(sun4v_vt_set_perfreg) retl nop ENDPROC(sun4v_vt_set_perfreg) + +ENTRY(sun4v_t5_get_perfreg) + mov %o1, %o4 + mov HV_FAST_T5_GET_PERFREG, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_t5_get_perfreg) + +ENTRY(sun4v_t5_set_perfreg) + mov HV_FAST_T5_SET_PERFREG, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_t5_set_perfreg) + +ENTRY(sun4v_m7_get_perfreg) + mov %o1, %o4 + mov HV_FAST_M7_GET_PERFREG, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_m7_get_perfreg) + +ENTRY(sun4v_m7_set_perfreg) + mov HV_FAST_M7_SET_PERFREG, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_m7_set_perfreg) + + /* %o0: address of CCB array + * %o1: size (in bytes) of CCB array + * %o2: flags + * %o3: reserved + * + * returns: + * %o0: status + * %o1: size (in bytes) of the CCB array that was accepted + * %o2: status data + * %o3: reserved + */ +ENTRY(sun4v_ccb_submit) + mov %o5, %g1 + mov HV_CCB_SUBMIT, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + stx %o2, [%g1] +ENDPROC(sun4v_ccb_submit) +EXPORT_SYMBOL(sun4v_ccb_submit) + + /* %o0: completion area ra for the ccb to get info + * + * returns: + * %o0: status + * %o1: CCB state + * %o2: position + * %o3: dax unit + * %o4: queue + */ +ENTRY(sun4v_ccb_info) + mov %o1, %g1 + mov HV_CCB_INFO, %o5 + ta HV_FAST_TRAP + sth %o1, [%g1 + CCB_INFO_OFFSET_CCB_STATE] + sth %o2, [%g1 + CCB_INFO_OFFSET_QUEUE_POS] + sth %o3, [%g1 + CCB_INFO_OFFSET_DAX_UNIT] + retl + sth %o4, [%g1 + CCB_INFO_OFFSET_QUEUE_NUM] +ENDPROC(sun4v_ccb_info) +EXPORT_SYMBOL(sun4v_ccb_info) + + /* %o0: completion area ra for the ccb to kill + * + * returns: + * %o0: status + * %o1: result of the kill + */ +ENTRY(sun4v_ccb_kill) + mov %o1, %g1 + mov HV_CCB_KILL, %o5 + ta HV_FAST_TRAP + retl + sth %o1, [%g1] +ENDPROC(sun4v_ccb_kill) +EXPORT_SYMBOL(sun4v_ccb_kill) diff --git a/arch/sparc/kernel/hvtramp.S b/arch/sparc/kernel/hvtramp.S index 605c960b2fa6..f39220471b65 100644 --- a/arch/sparc/kernel/hvtramp.S +++ b/arch/sparc/kernel/hvtramp.S @@ -1,9 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* hvtramp.S: Hypervisor start-cpu trampoline code. * * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> */ -#include <linux/init.h> #include <asm/thread_info.h> #include <asm/hypervisor.h> @@ -16,7 +16,6 @@ #include <asm/asi.h> #include <asm/pil.h> - __CPUINIT .align 8 .globl hv_cpu_startup, hv_cpu_startup_end @@ -111,7 +110,6 @@ hv_cpu_startup: sllx %g5, THREAD_SHIFT, %g5 sub %g5, (STACKFRAME_SZ + STACK_BIAS), %g5 add %g6, %g5, %sp - mov 0, %fp call init_irqwork_curcpu nop diff --git a/arch/sparc/kernel/idprom.c b/arch/sparc/kernel/idprom.c index 6bd75012109d..d6c46d512220 100644 --- a/arch/sparc/kernel/idprom.c +++ b/arch/sparc/kernel/idprom.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * idprom.c: Routines to load the idprom into kernel addresses and * interpret the data contained within. @@ -9,6 +10,7 @@ #include <linux/types.h> #include <linux/init.h> #include <linux/export.h> +#include <linux/etherdevice.h> #include <asm/oplib.h> #include <asm/idprom.h> @@ -60,6 +62,12 @@ static void __init display_system_type(unsigned char machtype) { } #endif + +unsigned char *arch_get_platform_mac_address(void) +{ + return idprom->id_ethaddr; +} + /* Calculate the IDPROM checksum (xor of the data bytes). */ static unsigned char __init calc_idprom_cksum(struct idprom *idprom) { diff --git a/arch/sparc/kernel/iommu-common.c b/arch/sparc/kernel/iommu-common.c new file mode 100644 index 000000000000..23ca75f09277 --- /dev/null +++ b/arch/sparc/kernel/iommu-common.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * IOMMU mmap management and range allocation functions. + * Based almost entirely upon the powerpc iommu allocator. + */ + +#include <linux/export.h> +#include <linux/bitmap.h> +#include <linux/bug.h> +#include <linux/iommu-helper.h> +#include <linux/dma-mapping.h> +#include <linux/hash.h> +#include <asm/iommu-common.h> + +static unsigned long iommu_large_alloc = 15; + +static DEFINE_PER_CPU(unsigned int, iommu_hash_common); + +static inline bool need_flush(struct iommu_map_table *iommu) +{ + return ((iommu->flags & IOMMU_NEED_FLUSH) != 0); +} + +static inline void set_flush(struct iommu_map_table *iommu) +{ + iommu->flags |= IOMMU_NEED_FLUSH; +} + +static inline void clear_flush(struct iommu_map_table *iommu) +{ + iommu->flags &= ~IOMMU_NEED_FLUSH; +} + +static void setup_iommu_pool_hash(void) +{ + unsigned int i; + static bool do_once; + + if (do_once) + return; + do_once = true; + for_each_possible_cpu(i) + per_cpu(iommu_hash_common, i) = hash_32(i, IOMMU_POOL_HASHBITS); +} + +/* + * Initialize iommu_pool entries for the iommu_map_table. `num_entries' + * is the number of table entries. If `large_pool' is set to true, + * the top 1/4 of the table will be set aside for pool allocations + * of more than iommu_large_alloc pages. + */ +void iommu_tbl_pool_init(struct iommu_map_table *iommu, + unsigned long num_entries, + u32 table_shift, + void (*lazy_flush)(struct iommu_map_table *), + bool large_pool, u32 npools, + bool skip_span_boundary_check) +{ + unsigned int start, i; + struct iommu_pool *p = &(iommu->large_pool); + + setup_iommu_pool_hash(); + if (npools == 0) + iommu->nr_pools = IOMMU_NR_POOLS; + else + iommu->nr_pools = npools; + BUG_ON(npools > IOMMU_NR_POOLS); + + iommu->table_shift = table_shift; + iommu->lazy_flush = lazy_flush; + start = 0; + if (skip_span_boundary_check) + iommu->flags |= IOMMU_NO_SPAN_BOUND; + if (large_pool) + iommu->flags |= IOMMU_HAS_LARGE_POOL; + + if (!large_pool) + iommu->poolsize = num_entries/iommu->nr_pools; + else + iommu->poolsize = (num_entries * 3 / 4)/iommu->nr_pools; + for (i = 0; i < iommu->nr_pools; i++) { + spin_lock_init(&(iommu->pools[i].lock)); + iommu->pools[i].start = start; + iommu->pools[i].hint = start; + start += iommu->poolsize; /* start for next pool */ + iommu->pools[i].end = start - 1; + } + if (!large_pool) + return; + /* initialize large_pool */ + spin_lock_init(&(p->lock)); + p->start = start; + p->hint = p->start; + p->end = num_entries; +} + +unsigned long iommu_tbl_range_alloc(struct device *dev, + struct iommu_map_table *iommu, + unsigned long npages, + unsigned long *handle, + unsigned long mask, + unsigned int align_order) +{ + unsigned int pool_hash = __this_cpu_read(iommu_hash_common); + unsigned long n, end, start, limit, boundary_size; + struct iommu_pool *pool; + int pass = 0; + unsigned int pool_nr; + unsigned int npools = iommu->nr_pools; + unsigned long flags; + bool large_pool = ((iommu->flags & IOMMU_HAS_LARGE_POOL) != 0); + bool largealloc = (large_pool && npages > iommu_large_alloc); + unsigned long shift; + unsigned long align_mask = 0; + + if (align_order > 0) + align_mask = ~0ul >> (BITS_PER_LONG - align_order); + + /* Sanity check */ + if (unlikely(npages == 0)) { + WARN_ON_ONCE(1); + return IOMMU_ERROR_CODE; + } + + if (largealloc) { + pool = &(iommu->large_pool); + pool_nr = 0; /* to keep compiler happy */ + } else { + /* pick out pool_nr */ + pool_nr = pool_hash & (npools - 1); + pool = &(iommu->pools[pool_nr]); + } + spin_lock_irqsave(&pool->lock, flags); + + again: + if (pass == 0 && handle && *handle && + (*handle >= pool->start) && (*handle < pool->end)) + start = *handle; + else + start = pool->hint; + + limit = pool->end; + + /* The case below can happen if we have a small segment appended + * to a large, or when the previous alloc was at the very end of + * the available space. If so, go back to the beginning. If a + * flush is needed, it will get done based on the return value + * from iommu_area_alloc() below. + */ + if (start >= limit) + start = pool->start; + shift = iommu->table_map_base >> iommu->table_shift; + if (limit + shift > mask) { + limit = mask - shift + 1; + /* If we're constrained on address range, first try + * at the masked hint to avoid O(n) search complexity, + * but on second pass, start at 0 in pool 0. + */ + if ((start & mask) >= limit || pass > 0) { + spin_unlock(&(pool->lock)); + pool = &(iommu->pools[0]); + spin_lock(&(pool->lock)); + start = pool->start; + } else { + start &= mask; + } + } + + /* + * if the skip_span_boundary_check had been set during init, we set + * things up so that iommu_is_span_boundary() merely checks if the + * (index + npages) < num_tsb_entries + */ + if ((iommu->flags & IOMMU_NO_SPAN_BOUND) != 0) { + shift = 0; + boundary_size = iommu->poolsize * iommu->nr_pools; + } else { + boundary_size = dma_get_seg_boundary_nr_pages(dev, + iommu->table_shift); + } + n = iommu_area_alloc(iommu->map, limit, start, npages, shift, + boundary_size, align_mask); + if (n == -1) { + if (likely(pass == 0)) { + /* First failure, rescan from the beginning. */ + pool->hint = pool->start; + set_flush(iommu); + pass++; + goto again; + } else if (!largealloc && pass <= iommu->nr_pools) { + spin_unlock(&(pool->lock)); + pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1); + pool = &(iommu->pools[pool_nr]); + spin_lock(&(pool->lock)); + pool->hint = pool->start; + set_flush(iommu); + pass++; + goto again; + } else { + /* give up */ + n = IOMMU_ERROR_CODE; + goto bail; + } + } + if (iommu->lazy_flush && + (n < pool->hint || need_flush(iommu))) { + clear_flush(iommu); + iommu->lazy_flush(iommu); + } + + end = n + npages; + pool->hint = end; + + /* Update handle for SG allocations */ + if (handle) + *handle = end; +bail: + spin_unlock_irqrestore(&(pool->lock), flags); + + return n; +} + +static struct iommu_pool *get_pool(struct iommu_map_table *tbl, + unsigned long entry) +{ + struct iommu_pool *p; + unsigned long largepool_start = tbl->large_pool.start; + bool large_pool = ((tbl->flags & IOMMU_HAS_LARGE_POOL) != 0); + + /* The large pool is the last pool at the top of the table */ + if (large_pool && entry >= largepool_start) { + p = &tbl->large_pool; + } else { + unsigned int pool_nr = entry / tbl->poolsize; + + BUG_ON(pool_nr >= tbl->nr_pools); + p = &tbl->pools[pool_nr]; + } + return p; +} + +/* Caller supplies the index of the entry into the iommu map table + * itself when the mapping from dma_addr to the entry is not the + * default addr->entry mapping below. + */ +void iommu_tbl_range_free(struct iommu_map_table *iommu, u64 dma_addr, + unsigned long npages, unsigned long entry) +{ + struct iommu_pool *pool; + unsigned long flags; + unsigned long shift = iommu->table_shift; + + if (entry == IOMMU_ERROR_CODE) /* use default addr->entry mapping */ + entry = (dma_addr - iommu->table_map_base) >> shift; + pool = get_pool(iommu, entry); + + spin_lock_irqsave(&(pool->lock), flags); + bitmap_clear(iommu->map, entry, npages); + spin_unlock_irqrestore(&(pool->lock), flags); +} diff --git a/arch/sparc/kernel/iommu.c b/arch/sparc/kernel/iommu.c index 070ed141aac7..46ef88bc9c26 100644 --- a/arch/sparc/kernel/iommu.c +++ b/arch/sparc/kernel/iommu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* iommu.c: Generic sparc64 IOMMU support. * * Copyright (C) 1999, 2007, 2008 David S. Miller (davem@davemloft.net) @@ -9,10 +10,11 @@ #include <linux/slab.h> #include <linux/delay.h> #include <linux/device.h> -#include <linux/dma-mapping.h> +#include <linux/dma-map-ops.h> #include <linux/errno.h> #include <linux/iommu-helper.h> #include <linux/bitmap.h> +#include <asm/iommu-common.h> #ifdef CONFIG_PCI #include <linux/pci.h> @@ -21,6 +23,7 @@ #include <asm/iommu.h> #include "iommu_common.h" +#include "kernel.h" #define STC_CTXMATCH_ADDR(STC, CTX) \ ((STC)->strbuf_ctxmatch_base + ((CTX) << 3)) @@ -44,8 +47,9 @@ "i" (ASI_PHYS_BYPASS_EC_E)) /* Must be invoked under the IOMMU lock. */ -static void iommu_flushall(struct iommu *iommu) +static void iommu_flushall(struct iommu_map_table *iommu_map_table) { + struct iommu *iommu = container_of(iommu_map_table, struct iommu, tbl); if (iommu->iommu_flushinv) { iommu_write(iommu->iommu_flushinv, ~(u64)0); } else { @@ -86,94 +90,6 @@ static inline void iopte_make_dummy(struct iommu *iommu, iopte_t *iopte) iopte_val(*iopte) = val; } -/* Based almost entirely upon the ppc64 iommu allocator. If you use the 'handle' - * facility it must all be done in one pass while under the iommu lock. - * - * On sun4u platforms, we only flush the IOMMU once every time we've passed - * over the entire page table doing allocations. Therefore we only ever advance - * the hint and cannot backtrack it. - */ -unsigned long iommu_range_alloc(struct device *dev, - struct iommu *iommu, - unsigned long npages, - unsigned long *handle) -{ - unsigned long n, end, start, limit, boundary_size; - struct iommu_arena *arena = &iommu->arena; - int pass = 0; - - /* This allocator was derived from x86_64's bit string search */ - - /* Sanity check */ - if (unlikely(npages == 0)) { - if (printk_ratelimit()) - WARN_ON(1); - return DMA_ERROR_CODE; - } - - if (handle && *handle) - start = *handle; - else - start = arena->hint; - - limit = arena->limit; - - /* The case below can happen if we have a small segment appended - * to a large, or when the previous alloc was at the very end of - * the available space. If so, go back to the beginning and flush. - */ - if (start >= limit) { - start = 0; - if (iommu->flush_all) - iommu->flush_all(iommu); - } - - again: - - if (dev) - boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, - 1 << IO_PAGE_SHIFT); - else - boundary_size = ALIGN(1UL << 32, 1 << IO_PAGE_SHIFT); - - n = iommu_area_alloc(arena->map, limit, start, npages, - iommu->page_table_map_base >> IO_PAGE_SHIFT, - boundary_size >> IO_PAGE_SHIFT, 0); - if (n == -1) { - if (likely(pass < 1)) { - /* First failure, rescan from the beginning. */ - start = 0; - if (iommu->flush_all) - iommu->flush_all(iommu); - pass++; - goto again; - } else { - /* Second failure, give up */ - return DMA_ERROR_CODE; - } - } - - end = n + npages; - - arena->hint = end; - - /* Update handle for SG allocations */ - if (handle) - *handle = end; - - return n; -} - -void iommu_range_free(struct iommu *iommu, dma_addr_t dma_addr, unsigned long npages) -{ - struct iommu_arena *arena = &iommu->arena; - unsigned long entry; - - entry = (dma_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT; - - bitmap_clear(arena->map, entry, npages); -} - int iommu_table_init(struct iommu *iommu, int tsbsize, u32 dma_offset, u32 dma_addr_mask, int numa_node) @@ -186,22 +102,19 @@ int iommu_table_init(struct iommu *iommu, int tsbsize, /* Setup initial software IOMMU state. */ spin_lock_init(&iommu->lock); iommu->ctx_lowest_free = 1; - iommu->page_table_map_base = dma_offset; + iommu->tbl.table_map_base = dma_offset; iommu->dma_addr_mask = dma_addr_mask; /* Allocate and initialize the free area map. */ sz = num_tsb_entries / 8; sz = (sz + 7UL) & ~7UL; - iommu->arena.map = kmalloc_node(sz, GFP_KERNEL, numa_node); - if (!iommu->arena.map) { - printk(KERN_ERR "IOMMU: Error, kmalloc(arena.map) failed.\n"); + iommu->tbl.map = kzalloc_node(sz, GFP_KERNEL, numa_node); + if (!iommu->tbl.map) return -ENOMEM; - } - memset(iommu->arena.map, 0, sz); - iommu->arena.limit = num_tsb_entries; - if (tlb_type != hypervisor) - iommu->flush_all = iommu_flushall; + iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT, + (tlb_type != hypervisor ? iommu_flushall : NULL), + false, 1, false); /* Allocate and initialize the dummy page which we * set inactive IO PTEs to point to. @@ -234,19 +147,21 @@ out_free_dummy_page: iommu->dummy_page = 0UL; out_free_map: - kfree(iommu->arena.map); - iommu->arena.map = NULL; + kfree(iommu->tbl.map); + iommu->tbl.map = NULL; return -ENOMEM; } -static inline iopte_t *alloc_npages(struct device *dev, struct iommu *iommu, +static inline iopte_t *alloc_npages(struct device *dev, + struct iommu *iommu, unsigned long npages) { unsigned long entry; - entry = iommu_range_alloc(dev, iommu, npages, NULL); - if (unlikely(entry == DMA_ERROR_CODE)) + entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL, + (unsigned long)(-1), 0); + if (unlikely(entry == IOMMU_ERROR_CODE)) return NULL; return iommu->page_table + entry; @@ -281,9 +196,9 @@ static inline void iommu_free_ctx(struct iommu *iommu, int ctx) static void *dma_4u_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { - unsigned long flags, order, first_page; + unsigned long order, first_page; struct iommu *iommu; struct page *page; int npages, nid; @@ -305,16 +220,14 @@ static void *dma_4u_alloc_coherent(struct device *dev, size_t size, iommu = dev->archdata.iommu; - spin_lock_irqsave(&iommu->lock, flags); iopte = alloc_npages(dev, iommu, size >> IO_PAGE_SHIFT); - spin_unlock_irqrestore(&iommu->lock, flags); if (unlikely(iopte == NULL)) { free_pages(first_page, order); return NULL; } - *dma_addrp = (iommu->page_table_map_base + + *dma_addrp = (iommu->tbl.table_map_base + ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); ret = (void *) first_page; npages = size >> IO_PAGE_SHIFT; @@ -332,50 +245,55 @@ static void *dma_4u_alloc_coherent(struct device *dev, size_t size, static void dma_4u_free_coherent(struct device *dev, size_t size, void *cpu, dma_addr_t dvma, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu *iommu; - unsigned long flags, order, npages; + unsigned long order, npages; npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; iommu = dev->archdata.iommu; - spin_lock_irqsave(&iommu->lock, flags); - - iommu_range_free(iommu, dvma, npages); - - spin_unlock_irqrestore(&iommu->lock, flags); + iommu_tbl_range_free(&iommu->tbl, dvma, npages, IOMMU_ERROR_CODE); order = get_order(size); if (order < 10) free_pages((unsigned long)cpu, order); } -static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t sz, - enum dma_data_direction direction, - struct dma_attrs *attrs) +static dma_addr_t dma_4u_map_phys(struct device *dev, phys_addr_t phys, + size_t sz, enum dma_data_direction direction, + unsigned long attrs) { struct iommu *iommu; struct strbuf *strbuf; iopte_t *base; unsigned long flags, npages, oaddr; - unsigned long i, base_paddr, ctx; + unsigned long i, ctx; u32 bus_addr, ret; unsigned long iopte_protection; + if (unlikely(attrs & DMA_ATTR_MMIO)) + /* + * This check is included because older versions of the code + * lacked MMIO path support, and my ability to test this path + * is limited. However, from a software technical standpoint, + * there is no restriction, as the following code operates + * solely on physical addresses. + */ + goto bad_no_ctx; + iommu = dev->archdata.iommu; strbuf = dev->archdata.stc; if (unlikely(direction == DMA_NONE)) goto bad_no_ctx; - oaddr = (unsigned long)(page_address(page) + offset); + oaddr = (unsigned long)(phys_to_virt(phys)); npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); npages >>= IO_PAGE_SHIFT; - spin_lock_irqsave(&iommu->lock, flags); base = alloc_npages(dev, iommu, npages); + spin_lock_irqsave(&iommu->lock, flags); ctx = 0; if (iommu->iommu_ctxflush) ctx = iommu_alloc_ctx(iommu); @@ -384,10 +302,9 @@ static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page, if (unlikely(!base)) goto bad; - bus_addr = (iommu->page_table_map_base + + bus_addr = (iommu->tbl.table_map_base + ((base - iommu->page_table) << IO_PAGE_SHIFT)); ret = bus_addr | (oaddr & ~IO_PAGE_MASK); - base_paddr = __pa(oaddr & IO_PAGE_MASK); if (strbuf->strbuf_enabled) iopte_protection = IOPTE_STREAMING(ctx); else @@ -395,8 +312,8 @@ static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page, if (direction != DMA_TO_DEVICE) iopte_protection |= IOPTE_WRITE; - for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE) - iopte_val(*base) = iopte_protection | base_paddr; + for (i = 0; i < npages; i++, base++, phys += IO_PAGE_SIZE) + iopte_val(*base) = iopte_protection | phys; return ret; @@ -405,7 +322,7 @@ bad: bad_no_ctx: if (printk_ratelimit()) WARN_ON(1); - return DMA_ERROR_CODE; + return DMA_MAPPING_ERROR; } static void strbuf_flush(struct strbuf *strbuf, struct iommu *iommu, @@ -474,9 +391,9 @@ do_flush_sync: vaddr, ctx, npages); } -static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, +static void dma_4u_unmap_phys(struct device *dev, dma_addr_t bus_addr, size_t sz, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu *iommu; struct strbuf *strbuf; @@ -495,7 +412,7 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); npages >>= IO_PAGE_SHIFT; base = iommu->page_table + - ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); + ((bus_addr - iommu->tbl.table_map_base) >> IO_PAGE_SHIFT); bus_addr &= IO_PAGE_MASK; spin_lock_irqsave(&iommu->lock, flags); @@ -506,7 +423,7 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL; /* Step 1: Kick data out of streaming buffers if necessary. */ - if (strbuf->strbuf_enabled) + if (strbuf->strbuf_enabled && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); @@ -514,16 +431,15 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, for (i = 0; i < npages; i++) iopte_make_dummy(iommu, base + i); - iommu_range_free(iommu, bus_addr, npages); - iommu_free_ctx(iommu, ctx); - spin_unlock_irqrestore(&iommu->lock, flags); + + iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, IOMMU_ERROR_CODE); } static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s, *outs, *segstart; unsigned long flags, handle, prot, ctx; @@ -540,7 +456,7 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, iommu = dev->archdata.iommu; strbuf = dev->archdata.stc; if (nelems == 0 || !iommu) - return 0; + return -EINVAL; spin_lock_irqsave(&iommu->lock, flags); @@ -564,9 +480,8 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, outs->dma_length = 0; max_seg_size = dma_get_max_seg_size(dev); - seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, - IO_PAGE_SIZE) >> IO_PAGE_SHIFT; - base_shift = iommu->page_table_map_base >> IO_PAGE_SHIFT; + seg_boundary_size = dma_get_seg_boundary_nr_pages(dev, IO_PAGE_SHIFT); + base_shift = iommu->tbl.table_map_base >> IO_PAGE_SHIFT; for_each_sg(sglist, s, nelems, i) { unsigned long paddr, npages, entry, out_entry = 0, slen; iopte_t *base; @@ -580,10 +495,11 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, /* Allocate iommu entries for that segment */ paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); - entry = iommu_range_alloc(dev, iommu, npages, &handle); + entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, + &handle, (unsigned long)(-1), 0); /* Handle failure */ - if (unlikely(entry == DMA_ERROR_CODE)) { + if (unlikely(entry == IOMMU_ERROR_CODE)) { if (printk_ratelimit()) printk(KERN_INFO "iommu_alloc failed, iommu %p paddr %lx" " npages %lx\n", iommu, paddr, npages); @@ -593,7 +509,7 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, base = iommu->page_table + entry; /* Convert entry to a dma_addr_t */ - dma_addr = iommu->page_table_map_base + + dma_addr = iommu->tbl.table_map_base + (entry << IO_PAGE_SHIFT); dma_addr |= (s->offset & ~IO_PAGE_MASK); @@ -638,7 +554,6 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, if (outcount < incount) { outs = sg_next(outs); - outs->dma_address = DMA_ERROR_CODE; outs->dma_length = 0; } @@ -653,16 +568,17 @@ iommu_map_failed: vaddr = s->dma_address & IO_PAGE_MASK; npages = iommu_num_pages(s->dma_address, s->dma_length, IO_PAGE_SIZE); - iommu_range_free(iommu, vaddr, npages); - entry = (vaddr - iommu->page_table_map_base) + entry = (vaddr - iommu->tbl.table_map_base) >> IO_PAGE_SHIFT; base = iommu->page_table + entry; for (j = 0; j < npages; j++) iopte_make_dummy(iommu, base + j); - s->dma_address = DMA_ERROR_CODE; + iommu_tbl_range_free(&iommu->tbl, vaddr, npages, + IOMMU_ERROR_CODE); + s->dma_length = 0; } if (s == outs) @@ -670,7 +586,7 @@ iommu_map_failed: } spin_unlock_irqrestore(&iommu->lock, flags); - return 0; + return -EINVAL; } /* If contexts are being used, they are the same in all of the mappings @@ -683,10 +599,11 @@ static unsigned long fetch_sg_ctx(struct iommu *iommu, struct scatterlist *sg) if (iommu->iommu_ctxflush) { iopte_t *base; u32 bus_addr; + struct iommu_map_table *tbl = &iommu->tbl; bus_addr = sg->dma_address & IO_PAGE_MASK; base = iommu->page_table + - ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); + ((bus_addr - tbl->table_map_base) >> IO_PAGE_SHIFT); ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL; } @@ -695,7 +612,7 @@ static unsigned long fetch_sg_ctx(struct iommu *iommu, struct scatterlist *sg) static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long flags, ctx; struct scatterlist *sg; @@ -722,20 +639,21 @@ static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist, if (!len) break; npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); - iommu_range_free(iommu, dma_handle, npages); - entry = ((dma_handle - iommu->page_table_map_base) + entry = ((dma_handle - iommu->tbl.table_map_base) >> IO_PAGE_SHIFT); base = iommu->page_table + entry; dma_handle &= IO_PAGE_MASK; - if (strbuf->strbuf_enabled) + if (strbuf->strbuf_enabled && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) strbuf_flush(strbuf, iommu, dma_handle, ctx, npages, direction); for (i = 0; i < npages; i++) iopte_make_dummy(iommu, base + i); + iommu_tbl_range_free(&iommu->tbl, dma_handle, npages, + IOMMU_ERROR_CODE); sg = sg_next(sg); } @@ -769,9 +687,10 @@ static void dma_4u_sync_single_for_cpu(struct device *dev, if (iommu->iommu_ctxflush && strbuf->strbuf_ctxflush) { iopte_t *iopte; + struct iommu_map_table *tbl = &iommu->tbl; iopte = iommu->page_table + - ((bus_addr - iommu->page_table_map_base)>>IO_PAGE_SHIFT); + ((bus_addr - tbl->table_map_base)>>IO_PAGE_SHIFT); ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; } @@ -804,9 +723,10 @@ static void dma_4u_sync_sg_for_cpu(struct device *dev, if (iommu->iommu_ctxflush && strbuf->strbuf_ctxflush) { iopte_t *iopte; + struct iommu_map_table *tbl = &iommu->tbl; - iopte = iommu->page_table + - ((sglist[0].dma_address - iommu->page_table_map_base) >> IO_PAGE_SHIFT); + iopte = iommu->page_table + ((sglist[0].dma_address - + tbl->table_map_base) >> IO_PAGE_SHIFT); ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; } @@ -826,38 +746,29 @@ static void dma_4u_sync_sg_for_cpu(struct device *dev, spin_unlock_irqrestore(&iommu->lock, flags); } -static struct dma_map_ops sun4u_dma_ops = { +static int dma_4u_supported(struct device *dev, u64 device_mask) +{ + struct iommu *iommu = dev->archdata.iommu; + + if (ali_sound_dma_hack(dev, device_mask)) + return 1; + + if (device_mask < iommu->dma_addr_mask) + return 0; + return 1; +} + +static const struct dma_map_ops sun4u_dma_ops = { .alloc = dma_4u_alloc_coherent, .free = dma_4u_free_coherent, - .map_page = dma_4u_map_page, - .unmap_page = dma_4u_unmap_page, + .map_phys = dma_4u_map_phys, + .unmap_phys = dma_4u_unmap_phys, .map_sg = dma_4u_map_sg, .unmap_sg = dma_4u_unmap_sg, .sync_single_for_cpu = dma_4u_sync_single_for_cpu, .sync_sg_for_cpu = dma_4u_sync_sg_for_cpu, + .dma_supported = dma_4u_supported, }; -struct dma_map_ops *dma_ops = &sun4u_dma_ops; +const struct dma_map_ops *dma_ops = &sun4u_dma_ops; EXPORT_SYMBOL(dma_ops); - -extern int pci64_dma_supported(struct pci_dev *pdev, u64 device_mask); - -int dma_supported(struct device *dev, u64 device_mask) -{ - struct iommu *iommu = dev->archdata.iommu; - u64 dma_addr_mask = iommu->dma_addr_mask; - - if (device_mask >= (1UL << 32UL)) - return 0; - - if ((device_mask & dma_addr_mask) == dma_addr_mask) - return 1; - -#ifdef CONFIG_PCI - if (dev->bus == &pci_bus_type) - return pci64_dma_supported(to_pci_dev(dev), device_mask); -#endif - - return 0; -} -EXPORT_SYMBOL(dma_supported); diff --git a/arch/sparc/kernel/iommu_common.h b/arch/sparc/kernel/iommu_common.h index 591f5879039c..d62ed9c5682d 100644 --- a/arch/sparc/kernel/iommu_common.h +++ b/arch/sparc/kernel/iommu_common.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* iommu_common.h: UltraSparc SBUS/PCI common iommu declarations. * * Copyright (C) 1999, 2008 David S. Miller (davem@davemloft.net) @@ -15,7 +16,6 @@ #include <linux/iommu-helper.h> #include <asm/iommu.h> -#include <asm/scatterlist.h> /* * These give mapping size of each iommu pte/tlb. @@ -48,12 +48,4 @@ static inline int is_span_boundary(unsigned long entry, return iommu_is_span_boundary(entry, nr, shift, boundary_size); } -extern unsigned long iommu_range_alloc(struct device *dev, - struct iommu *iommu, - unsigned long npages, - unsigned long *handle); -extern void iommu_range_free(struct iommu *iommu, - dma_addr_t dma_addr, - unsigned long npages); - #endif /* _IOMMU_COMMON_H */ diff --git a/arch/sparc/kernel/ioport.c b/arch/sparc/kernel/ioport.c index 2096468de9b2..5ebca5c7af1e 100644 --- a/arch/sparc/kernel/ioport.c +++ b/arch/sparc/kernel/ioport.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * ioport.c: Simple io mapping allocator. * @@ -37,7 +38,8 @@ #include <linux/proc_fs.h> #include <linux/seq_file.h> #include <linux/scatterlist.h> -#include <linux/of_device.h> +#include <linux/dma-map-ops.h> +#include <linux/of.h> #include <asm/io.h> #include <asm/vaddrs.h> @@ -50,19 +52,6 @@ #include <asm/io-unit.h> #include <asm/leon.h> -const struct sparc32_dma_ops *sparc32_dma_ops; - -/* This function must make sure that caches and memory are coherent after DMA - * On LEON systems without cache snooping it flushes the entire D-CACHE. - */ -static inline void dma_make_coherent(unsigned long pa, unsigned long len) -{ - if (sparc_cpu_model == sparc_leon) { - if (!sparc_leon3_snooping_enabled()) - leon_flush_dcache_all(); - } -} - static void __iomem *_sparc_ioremap(struct resource *res, u32 bus, u32 pa, int sz); static void __iomem *_sparc_alloc_io(unsigned int busno, unsigned long phys, unsigned long size, char *name); @@ -121,17 +110,17 @@ static void xres_free(struct xresource *xrp) { * * Bus type is always zero on IIep. */ -void __iomem *ioremap(unsigned long offset, unsigned long size) +void __iomem *ioremap(phys_addr_t offset, size_t size) { char name[14]; sprintf(name, "phys_%08x", (u32)offset); - return _sparc_alloc_io(0, offset, size, name); + return _sparc_alloc_io(0, (unsigned long)offset, size, name); } EXPORT_SYMBOL(ioremap); /* - * Comlimentary to ioremap(). + * Complementary to ioremap(). */ void iounmap(volatile void __iomem *virtual) { @@ -186,7 +175,7 @@ static void __iomem *_sparc_alloc_io(unsigned int busno, unsigned long phys, if (name == NULL) name = "???"; - if ((xres = xres_alloc()) != 0) { + if ((xres = xres_alloc()) != NULL) { tack = xres->xname; res = &xres->xres; } else { @@ -202,7 +191,7 @@ static void __iomem *_sparc_alloc_io(unsigned int busno, unsigned long phys, tack += sizeof (struct resource); } - strlcpy(tack, name, XNMLN+1); + strscpy(tack, name, XNMLN+1); res->name = tack; va = _sparc_ioremap(res, busno, phys, size); @@ -233,7 +222,7 @@ _sparc_ioremap(struct resource *res, u32 bus, u32 pa, int sz) } /* - * Comlimentary to _sparc_ioremap(). + * Complementary to _sparc_ioremap(). */ static void _sparc_free_io(struct resource *res) { @@ -245,171 +234,60 @@ static void _sparc_free_io(struct resource *res) release_resource(res); } -#ifdef CONFIG_SBUS - -void sbus_set_sbus64(struct device *dev, int x) +unsigned long sparc_dma_alloc_resource(struct device *dev, size_t len) { - printk("sbus_set_sbus64: unsupported\n"); -} -EXPORT_SYMBOL(sbus_set_sbus64); - -/* - * Allocate a chunk of memory suitable for DMA. - * Typically devices use them for control blocks. - * CPU may access them without any explicit flushing. - */ -static void *sbus_alloc_coherent(struct device *dev, size_t len, - dma_addr_t *dma_addrp, gfp_t gfp, - struct dma_attrs *attrs) -{ - struct platform_device *op = to_platform_device(dev); - unsigned long len_total = PAGE_ALIGN(len); - unsigned long va; struct resource *res; - int order; - /* XXX why are some lengths signed, others unsigned? */ - if (len <= 0) { - return NULL; - } - /* XXX So what is maxphys for us and how do drivers know it? */ - if (len > 256*1024) { /* __get_free_pages() limit */ - return NULL; - } - - order = get_order(len_total); - if ((va = __get_free_pages(GFP_KERNEL|__GFP_COMP, order)) == 0) - goto err_nopages; - - if ((res = kzalloc(sizeof(struct resource), GFP_KERNEL)) == NULL) - goto err_nomem; + res = kzalloc(sizeof(*res), GFP_KERNEL); + if (!res) + return 0; + res->name = dev->of_node->full_name; - if (allocate_resource(&_sparc_dvma, res, len_total, - _sparc_dvma.start, _sparc_dvma.end, PAGE_SIZE, NULL, NULL) != 0) { - printk("sbus_alloc_consistent: cannot occupy 0x%lx", len_total); - goto err_nova; + if (allocate_resource(&_sparc_dvma, res, len, _sparc_dvma.start, + _sparc_dvma.end, PAGE_SIZE, NULL, NULL) != 0) { + printk("%s: cannot occupy 0x%zx", __func__, len); + kfree(res); + return 0; } - // XXX The sbus_map_dma_area does this for us below, see comments. - // srmmu_mapiorange(0, virt_to_phys(va), res->start, len_total); - /* - * XXX That's where sdev would be used. Currently we load - * all iommu tables with the same translations. - */ - if (sbus_map_dma_area(dev, dma_addrp, va, res->start, len_total) != 0) - goto err_noiommu; - - res->name = op->dev.of_node->name; - - return (void *)(unsigned long)res->start; - -err_noiommu: - release_resource(res); -err_nova: - kfree(res); -err_nomem: - free_pages(va, order); -err_nopages: - return NULL; + return res->start; } -static void sbus_free_coherent(struct device *dev, size_t n, void *p, - dma_addr_t ba, struct dma_attrs *attrs) +bool sparc_dma_free_resource(void *cpu_addr, size_t size) { + unsigned long addr = (unsigned long)cpu_addr; struct resource *res; - struct page *pgv; - if ((res = lookup_resource(&_sparc_dvma, - (unsigned long)p)) == NULL) { - printk("sbus_free_consistent: cannot free %p\n", p); - return; + res = lookup_resource(&_sparc_dvma, addr); + if (!res) { + printk("%s: cannot free %p\n", __func__, cpu_addr); + return false; } - if (((unsigned long)p & (PAGE_SIZE-1)) != 0) { - printk("sbus_free_consistent: unaligned va %p\n", p); - return; + if ((addr & (PAGE_SIZE - 1)) != 0) { + printk("%s: unaligned va %p\n", __func__, cpu_addr); + return false; } - n = PAGE_ALIGN(n); - if (resource_size(res) != n) { - printk("sbus_free_consistent: region 0x%lx asked 0x%zx\n", - (long)resource_size(res), n); - return; + size = PAGE_ALIGN(size); + if (resource_size(res) != size) { + printk("%s: region 0x%lx asked 0x%zx\n", + __func__, (long)resource_size(res), size); + return false; } release_resource(res); kfree(res); - - pgv = virt_to_page(p); - sbus_unmap_dma_area(dev, ba, n); - - __free_pages(pgv, get_order(n)); + return true; } -/* - * Map a chunk of memory so that devices can see it. - * CPU view of this memory may be inconsistent with - * a device view and explicit flushing is necessary. - */ -static dma_addr_t sbus_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t len, - enum dma_data_direction dir, - struct dma_attrs *attrs) -{ - void *va = page_address(page) + offset; - - /* XXX why are some lengths signed, others unsigned? */ - if (len <= 0) { - return 0; - } - /* XXX So what is maxphys for us and how do drivers know it? */ - if (len > 256*1024) { /* __get_free_pages() limit */ - return 0; - } - return mmu_get_scsi_one(dev, va, len); -} - -static void sbus_unmap_page(struct device *dev, dma_addr_t ba, size_t n, - enum dma_data_direction dir, struct dma_attrs *attrs) -{ - mmu_release_scsi_one(dev, ba, n); -} - -static int sbus_map_sg(struct device *dev, struct scatterlist *sg, int n, - enum dma_data_direction dir, struct dma_attrs *attrs) -{ - mmu_get_scsi_sgl(dev, sg, n); - return n; -} - -static void sbus_unmap_sg(struct device *dev, struct scatterlist *sg, int n, - enum dma_data_direction dir, struct dma_attrs *attrs) -{ - mmu_release_scsi_sgl(dev, sg, n); -} - -static void sbus_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, - int n, enum dma_data_direction dir) -{ - BUG(); -} +#ifdef CONFIG_SBUS -static void sbus_sync_sg_for_device(struct device *dev, struct scatterlist *sg, - int n, enum dma_data_direction dir) +void sbus_set_sbus64(struct device *dev, int x) { - BUG(); + printk("sbus_set_sbus64: unsupported\n"); } - -struct dma_map_ops sbus_dma_ops = { - .alloc = sbus_alloc_coherent, - .free = sbus_free_coherent, - .map_page = sbus_map_page, - .unmap_page = sbus_unmap_page, - .map_sg = sbus_map_sg, - .unmap_sg = sbus_unmap_sg, - .sync_sg_for_cpu = sbus_sync_sg_for_cpu, - .sync_sg_for_device = sbus_sync_sg_for_device, -}; +EXPORT_SYMBOL(sbus_set_sbus64); static int __init sparc_register_ioport(void) { @@ -422,258 +300,21 @@ arch_initcall(sparc_register_ioport); #endif /* CONFIG_SBUS */ - -/* Allocate and map kernel buffer using consistent mode DMA for a device. - * hwdev should be valid struct pci_dev pointer for PCI devices. - */ -static void *pci32_alloc_coherent(struct device *dev, size_t len, - dma_addr_t *pba, gfp_t gfp, - struct dma_attrs *attrs) -{ - unsigned long len_total = PAGE_ALIGN(len); - void *va; - struct resource *res; - int order; - - if (len == 0) { - return NULL; - } - if (len > 256*1024) { /* __get_free_pages() limit */ - return NULL; - } - - order = get_order(len_total); - va = (void *) __get_free_pages(GFP_KERNEL, order); - if (va == NULL) { - printk("pci_alloc_consistent: no %ld pages\n", len_total>>PAGE_SHIFT); - goto err_nopages; - } - - if ((res = kzalloc(sizeof(struct resource), GFP_KERNEL)) == NULL) { - printk("pci_alloc_consistent: no core\n"); - goto err_nomem; - } - - if (allocate_resource(&_sparc_dvma, res, len_total, - _sparc_dvma.start, _sparc_dvma.end, PAGE_SIZE, NULL, NULL) != 0) { - printk("pci_alloc_consistent: cannot occupy 0x%lx", len_total); - goto err_nova; - } - srmmu_mapiorange(0, virt_to_phys(va), res->start, len_total); - - *pba = virt_to_phys(va); /* equals virt_to_bus (R.I.P.) for us. */ - return (void *) res->start; - -err_nova: - kfree(res); -err_nomem: - free_pages((unsigned long)va, order); -err_nopages: - return NULL; -} - -/* Free and unmap a consistent DMA buffer. - * cpu_addr is what was returned from pci_alloc_consistent, - * size must be the same as what as passed into pci_alloc_consistent, - * and likewise dma_addr must be the same as what *dma_addrp was set to. - * - * References to the memory and mappings associated with cpu_addr/dma_addr - * past this call are illegal. - */ -static void pci32_free_coherent(struct device *dev, size_t n, void *p, - dma_addr_t ba, struct dma_attrs *attrs) -{ - struct resource *res; - - if ((res = lookup_resource(&_sparc_dvma, - (unsigned long)p)) == NULL) { - printk("pci_free_consistent: cannot free %p\n", p); - return; - } - - if (((unsigned long)p & (PAGE_SIZE-1)) != 0) { - printk("pci_free_consistent: unaligned va %p\n", p); - return; - } - - n = PAGE_ALIGN(n); - if (resource_size(res) != n) { - printk("pci_free_consistent: region 0x%lx asked 0x%lx\n", - (long)resource_size(res), (long)n); - return; - } - - dma_make_coherent(ba, n); - srmmu_unmapiorange((unsigned long)p, n); - - release_resource(res); - kfree(res); - free_pages((unsigned long)phys_to_virt(ba), get_order(n)); -} - /* - * Same as pci_map_single, but with pages. - */ -static dma_addr_t pci32_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, - struct dma_attrs *attrs) -{ - /* IIep is write-through, not flushing. */ - return page_to_phys(page) + offset; -} - -static void pci32_unmap_page(struct device *dev, dma_addr_t ba, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) -{ - if (dir != PCI_DMA_TODEVICE) - dma_make_coherent(ba, PAGE_ALIGN(size)); -} - -/* Map a set of buffers described by scatterlist in streaming - * mode for DMA. This is the scather-gather version of the - * above pci_map_single interface. Here the scatter gather list - * elements are each tagged with the appropriate dma address - * and length. They are obtained via sg_dma_{address,length}(SG). + * IIep is write-through, not flushing on cpu to device transfer. * - * NOTE: An implementation may be able to use a smaller number of - * DMA address/length pairs than there are SG table elements. - * (for example via virtual mapping capabilities) - * The routine returns the number of addr/length pairs actually - * used, at most nents. - * - * Device ownership issues as mentioned above for pci_map_single are - * the same here. - */ -static int pci32_map_sg(struct device *device, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) -{ - struct scatterlist *sg; - int n; - - /* IIep is write-through, not flushing. */ - for_each_sg(sgl, sg, nents, n) { - sg->dma_address = sg_phys(sg); - sg->dma_length = sg->length; - } - return nents; -} - -/* Unmap a set of streaming mode DMA translations. - * Again, cpu read rules concerning calls here are the same as for - * pci_unmap_single() above. + * On LEON systems without cache snooping, the entire D-CACHE must be flushed to + * make DMA to cacheable memory coherent. */ -static void pci32_unmap_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { - struct scatterlist *sg; - int n; - - if (dir != PCI_DMA_TODEVICE) { - for_each_sg(sgl, sg, nents, n) { - dma_make_coherent(sg_phys(sg), PAGE_ALIGN(sg->length)); - } - } + if (dir != DMA_TO_DEVICE && + sparc_cpu_model == sparc_leon && + !sparc_leon3_snooping_enabled()) + leon_flush_dcache_all(); } -/* Make physical memory consistent for a single - * streaming mode DMA translation before or after a transfer. - * - * If you perform a pci_map_single() but wish to interrogate the - * buffer using the cpu, yet do not wish to teardown the PCI dma - * mapping, you must call this function before doing so. At the - * next point you give the PCI dma address back to the card, you - * must first perform a pci_dma_sync_for_device, and then the - * device again owns the buffer. - */ -static void pci32_sync_single_for_cpu(struct device *dev, dma_addr_t ba, - size_t size, enum dma_data_direction dir) -{ - if (dir != PCI_DMA_TODEVICE) { - dma_make_coherent(ba, PAGE_ALIGN(size)); - } -} - -static void pci32_sync_single_for_device(struct device *dev, dma_addr_t ba, - size_t size, enum dma_data_direction dir) -{ - if (dir != PCI_DMA_TODEVICE) { - dma_make_coherent(ba, PAGE_ALIGN(size)); - } -} - -/* Make physical memory consistent for a set of streaming - * mode DMA translations after a transfer. - * - * The same as pci_dma_sync_single_* but for a scatter-gather list, - * same rules and usage. - */ -static void pci32_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir) -{ - struct scatterlist *sg; - int n; - - if (dir != PCI_DMA_TODEVICE) { - for_each_sg(sgl, sg, nents, n) { - dma_make_coherent(sg_phys(sg), PAGE_ALIGN(sg->length)); - } - } -} - -static void pci32_sync_sg_for_device(struct device *device, struct scatterlist *sgl, - int nents, enum dma_data_direction dir) -{ - struct scatterlist *sg; - int n; - - if (dir != PCI_DMA_TODEVICE) { - for_each_sg(sgl, sg, nents, n) { - dma_make_coherent(sg_phys(sg), PAGE_ALIGN(sg->length)); - } - } -} - -struct dma_map_ops pci32_dma_ops = { - .alloc = pci32_alloc_coherent, - .free = pci32_free_coherent, - .map_page = pci32_map_page, - .unmap_page = pci32_unmap_page, - .map_sg = pci32_map_sg, - .unmap_sg = pci32_unmap_sg, - .sync_single_for_cpu = pci32_sync_single_for_cpu, - .sync_single_for_device = pci32_sync_single_for_device, - .sync_sg_for_cpu = pci32_sync_sg_for_cpu, - .sync_sg_for_device = pci32_sync_sg_for_device, -}; -EXPORT_SYMBOL(pci32_dma_ops); - -/* leon re-uses pci32_dma_ops */ -struct dma_map_ops *leon_dma_ops = &pci32_dma_ops; -EXPORT_SYMBOL(leon_dma_ops); - -struct dma_map_ops *dma_ops = &sbus_dma_ops; -EXPORT_SYMBOL(dma_ops); - - -/* - * Return whether the given PCI device DMA address mask can be - * supported properly. For example, if your device can only drive the - * low 24-bits during PCI bus mastering, then you would pass - * 0x00ffffff as the mask to this function. - */ -int dma_supported(struct device *dev, u64 mask) -{ -#ifdef CONFIG_PCI - if (dev->bus == &pci_bus_type) - return 1; -#endif - return 0; -} -EXPORT_SYMBOL(dma_supported); - #ifdef CONFIG_PROC_FS static int sparc_io_proc_show(struct seq_file *m, void *v) @@ -682,7 +323,7 @@ static int sparc_io_proc_show(struct seq_file *m, void *v) const char *nm; for (r = root->child; r != NULL; r = r->sibling) { - if ((nm = r->name) == 0) nm = "???"; + if ((nm = r->name) == NULL) nm = "???"; seq_printf(m, "%016llx-%016llx: %s\n", (unsigned long long)r->start, (unsigned long long)r->end, nm); @@ -690,25 +331,14 @@ static int sparc_io_proc_show(struct seq_file *m, void *v) return 0; } - -static int sparc_io_proc_open(struct inode *inode, struct file *file) -{ - return single_open(file, sparc_io_proc_show, PDE_DATA(inode)); -} - -static const struct file_operations sparc_io_proc_fops = { - .owner = THIS_MODULE, - .open = sparc_io_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; #endif /* CONFIG_PROC_FS */ static void register_proc_sparc_ioport(void) { #ifdef CONFIG_PROC_FS - proc_create_data("io_map", 0, NULL, &sparc_io_proc_fops, &sparc_iomap); - proc_create_data("dvma_map", 0, NULL, &sparc_io_proc_fops, &_sparc_dvma); + proc_create_single_data("io_map", 0, NULL, sparc_io_proc_show, + &sparc_iomap); + proc_create_single_data("dvma_map", 0, NULL, sparc_io_proc_show, + &_sparc_dvma); #endif } diff --git a/arch/sparc/kernel/irq.h b/arch/sparc/kernel/irq.h index b66b6aad1d6d..b02026ad6e34 100644 --- a/arch/sparc/kernel/irq.h +++ b/arch/sparc/kernel/irq.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/platform_device.h> #include <asm/cpu_type.h> @@ -82,11 +83,20 @@ void handler_irq(unsigned int pil, struct pt_regs *regs); unsigned long leon_get_irqmask(unsigned int irq); +/* irq_32.c */ +void sparc_floppy_irq(int irq, void *dev_id, struct pt_regs *regs); + +/* sun4m_irq.c */ +void sun4m_nmi(struct pt_regs *regs); + +/* sun4d_irq.c */ +void sun4d_handler_irq(unsigned int pil, struct pt_regs *regs); + #ifdef CONFIG_SMP /* All SUN4D IPIs are sent on this IRQ, may be shared with hard IRQs */ #define SUN4D_IPI_IRQ 13 -extern void sun4d_ipi_interrupt(void); +void sun4d_ipi_interrupt(void); #endif diff --git a/arch/sparc/kernel/irq_32.c b/arch/sparc/kernel/irq_32.c index c145f6fd123b..5210991429d5 100644 --- a/arch/sparc/kernel/irq_32.c +++ b/arch/sparc/kernel/irq_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Interrupt request handling routines. On the * Sparc the IRQs are basically 'cast in stone' @@ -17,6 +18,7 @@ #include <asm/cacheflush.h> #include <asm/cpudata.h> +#include <asm/setup.h> #include <asm/pcic.h> #include <asm/leon.h> @@ -164,7 +166,7 @@ void irq_link(unsigned int irq) p = &irq_table[irq]; pil = p->pil; - BUG_ON(pil > SUN4D_MAX_IRQ); + BUG_ON(pil >= SUN4D_MAX_IRQ); p->next = irq_map[pil]; irq_map[pil] = p; @@ -181,7 +183,7 @@ void irq_unlink(unsigned int irq) spin_lock_irqsave(&irq_map_lock, flags); p = &irq_table[irq]; - BUG_ON(p->pil > SUN4D_MAX_IRQ); + BUG_ON(p->pil >= SUN4D_MAX_IRQ); pnext = &irq_map[p->pil]; while (*pnext != p) pnext = &(*pnext)->next; @@ -197,18 +199,18 @@ int arch_show_interrupts(struct seq_file *p, int prec) int j; #ifdef CONFIG_SMP - seq_printf(p, "RES: "); + seq_printf(p, "RES:"); for_each_online_cpu(j) - seq_printf(p, "%10u ", cpu_data(j).irq_resched_count); + seq_put_decimal_ull_width(p, " ", cpu_data(j).irq_resched_count, 10); seq_printf(p, " IPI rescheduling interrupts\n"); - seq_printf(p, "CAL: "); + seq_printf(p, "CAL:"); for_each_online_cpu(j) - seq_printf(p, "%10u ", cpu_data(j).irq_call_count); + seq_put_decimal_ull_width(p, " ", cpu_data(j).irq_call_count, 10); seq_printf(p, " IPI function call interrupts\n"); #endif - seq_printf(p, "NMI: "); + seq_printf(p, "NMI:"); for_each_online_cpu(j) - seq_printf(p, "%10u ", cpu_data(j).counter); + seq_put_decimal_ull_width(p, " ", cpu_data(j).counter, 10); seq_printf(p, " Non-maskable interrupts\n"); return 0; } @@ -266,11 +268,11 @@ int sparc_floppy_request_irq(unsigned int irq, irq_handler_t irq_handler) if (sparc_cpu_model != sparc_leon) { struct tt_entry *trap_table; - trap_table = &trapbase_cpu1; + trap_table = &trapbase_cpu1[0]; INSTANTIATE(trap_table) - trap_table = &trapbase_cpu2; + trap_table = &trapbase_cpu2[0]; INSTANTIATE(trap_table) - trap_table = &trapbase_cpu3; + trap_table = &trapbase_cpu3[0]; INSTANTIATE(trap_table) } #endif diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index 9bcbbe2c4e7e..ded463c82abd 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* irq.c: UltraSparc IRQ handling/init/registry. * * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net) @@ -21,7 +22,7 @@ #include <linux/seq_file.h> #include <linux/ftrace.h> #include <linux/irq.h> -#include <linux/kmemleak.h> +#include <linux/string_choices.h> #include <asm/ptrace.h> #include <asm/processor.h> @@ -35,20 +36,19 @@ #include <asm/timer.h> #include <asm/smp.h> #include <asm/starfire.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/cache.h> #include <asm/cpudata.h> #include <asm/auxio.h> #include <asm/head.h> #include <asm/hypervisor.h> #include <asm/cacheflush.h> +#include <asm/softirq_stack.h> #include "entry.h" #include "cpumap.h" #include "kstack.h" -#define NUM_IVECS (IMAP_INR + 1) - struct ino_bucket *ivector_table; unsigned long ivector_table_pa; @@ -107,55 +107,194 @@ static void bucket_set_irq(unsigned long bucket_pa, unsigned int irq) #define irq_work_pa(__cpu) &(trap_block[(__cpu)].irq_worklist_pa) -static struct { - unsigned int dev_handle; - unsigned int dev_ino; - unsigned int in_use; -} irq_table[NR_IRQS]; -static DEFINE_SPINLOCK(irq_alloc_lock); +static unsigned long hvirq_major __initdata; +static int __init early_hvirq_major(char *p) +{ + int rc = kstrtoul(p, 10, &hvirq_major); + + return rc; +} +early_param("hvirq", early_hvirq_major); + +static int hv_irq_version; + +/* Major version 2.0 of HV_GRP_INTR added support for the VIRQ cookie + * based interfaces, but: + * + * 1) Several OSs, Solaris and Linux included, use them even when only + * negotiating version 1.0 (or failing to negotiate at all). So the + * hypervisor has a workaround that provides the VIRQ interfaces even + * when only verion 1.0 of the API is in use. + * + * 2) Second, and more importantly, with major version 2.0 these VIRQ + * interfaces only were actually hooked up for LDC interrupts, even + * though the Hypervisor specification clearly stated: + * + * The new interrupt API functions will be available to a guest + * when it negotiates version 2.0 in the interrupt API group 0x2. When + * a guest negotiates version 2.0, all interrupt sources will only + * support using the cookie interface, and any attempt to use the + * version 1.0 interrupt APIs numbered 0xa0 to 0xa6 will result in the + * ENOTSUPPORTED error being returned. + * + * with an emphasis on "all interrupt sources". + * + * To correct this, major version 3.0 was created which does actually + * support VIRQs for all interrupt sources (not just LDC devices). So + * if we want to move completely over the cookie based VIRQs we must + * negotiate major version 3.0 or later of HV_GRP_INTR. + */ +static bool sun4v_cookie_only_virqs(void) +{ + return hv_irq_version >= 3; +} -unsigned char irq_alloc(unsigned int dev_handle, unsigned int dev_ino) +static void __init irq_init_hv(void) { - unsigned long flags; - unsigned char ent; + unsigned long hv_error, major, minor = 0; - BUILD_BUG_ON(NR_IRQS >= 256); + if (tlb_type != hypervisor) + return; + + if (hvirq_major) + major = hvirq_major; + else + major = 3; + + hv_error = sun4v_hvapi_register(HV_GRP_INTR, major, &minor); + if (!hv_error) + hv_irq_version = major; + else + hv_irq_version = 1; + + pr_info("SUN4V: Using IRQ API major %d, cookie only virqs %s\n", + hv_irq_version, + str_enabled_disabled(sun4v_cookie_only_virqs())); +} + +/* This function is for the timer interrupt.*/ +int __init arch_probe_nr_irqs(void) +{ + return 1; +} - spin_lock_irqsave(&irq_alloc_lock, flags); +#define DEFAULT_NUM_IVECS (0xfffU) +static unsigned int nr_ivec = DEFAULT_NUM_IVECS; +#define NUM_IVECS (nr_ivec) - for (ent = 1; ent < NR_IRQS; ent++) { - if (!irq_table[ent].in_use) +static unsigned int __init size_nr_ivec(void) +{ + if (tlb_type == hypervisor) { + switch (sun4v_chip_type) { + /* Athena's devhandle|devino is large.*/ + case SUN4V_CHIP_SPARC64X: + nr_ivec = 0xffff; break; + } } - if (ent >= NR_IRQS) { - printk(KERN_ERR "IRQ: Out of virtual IRQs.\n"); - ent = 0; - } else { - irq_table[ent].dev_handle = dev_handle; - irq_table[ent].dev_ino = dev_ino; - irq_table[ent].in_use = 1; - } + return nr_ivec; +} - spin_unlock_irqrestore(&irq_alloc_lock, flags); +struct irq_handler_data { + union { + struct { + unsigned int dev_handle; + unsigned int dev_ino; + }; + unsigned long sysino; + }; + struct ino_bucket bucket; + unsigned long iclr; + unsigned long imap; +}; + +static inline unsigned int irq_data_to_handle(struct irq_data *data) +{ + struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data); + + return ihd->dev_handle; +} + +static inline unsigned int irq_data_to_ino(struct irq_data *data) +{ + struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data); + + return ihd->dev_ino; +} + +static inline unsigned long irq_data_to_sysino(struct irq_data *data) +{ + struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data); - return ent; + return ihd->sysino; } -#ifdef CONFIG_PCI_MSI void irq_free(unsigned int irq) { - unsigned long flags; + void *data = irq_get_handler_data(irq); - if (irq >= NR_IRQS) - return; + kfree(data); + irq_set_handler_data(irq, NULL); + irq_free_descs(irq, 1); +} - spin_lock_irqsave(&irq_alloc_lock, flags); +unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino) +{ + int irq; - irq_table[irq].in_use = 0; + irq = __irq_alloc_descs(-1, 1, 1, numa_node_id(), NULL, NULL); + if (irq <= 0) + goto out; - spin_unlock_irqrestore(&irq_alloc_lock, flags); + return irq; +out: + return 0; +} + +static unsigned int cookie_exists(u32 devhandle, unsigned int devino) +{ + unsigned long hv_err, cookie; + struct ino_bucket *bucket; + unsigned int irq = 0U; + + hv_err = sun4v_vintr_get_cookie(devhandle, devino, &cookie); + if (hv_err) { + pr_err("HV get cookie failed hv_err = %ld\n", hv_err); + goto out; + } + + if (cookie & ((1UL << 63UL))) { + cookie = ~cookie; + bucket = (struct ino_bucket *) __va(cookie); + irq = bucket->__irq; + } +out: + return irq; +} + +static unsigned int sysino_exists(u32 devhandle, unsigned int devino) +{ + unsigned long sysino = sun4v_devino_to_sysino(devhandle, devino); + struct ino_bucket *bucket; + unsigned int irq; + + bucket = &ivector_table[sysino]; + irq = bucket_get_irq(__pa(bucket)); + + return irq; +} + +void ack_bad_irq(unsigned int irq) +{ + pr_crit("BAD IRQ ack %d\n", irq); +} + +void irq_install_pre_handler(int irq, + void (*func)(unsigned int, void *, void *), + void *arg1, void *arg2) +{ + pr_warn("IRQ pre handler NOT supported.\n"); } -#endif /* * /proc/interrupts printing: @@ -164,9 +303,9 @@ int arch_show_interrupts(struct seq_file *p, int prec) { int j; - seq_printf(p, "NMI: "); + seq_printf(p, "NMI:"); for_each_online_cpu(j) - seq_printf(p, "%10u ", cpu_data(j).__nmi_count); + seq_put_decimal_ull_width(p, " ", cpu_data(j).__nmi_count, 10); seq_printf(p, " Non-maskable interrupts\n"); return 0; } @@ -206,29 +345,16 @@ static unsigned int sun4u_compute_tid(unsigned long imap, unsigned long cpuid) return tid; } -struct irq_handler_data { - unsigned long iclr; - unsigned long imap; - - void (*pre_handler)(unsigned int, void *, void *); - void *arg1; - void *arg2; -}; - #ifdef CONFIG_SMP static int irq_choose_cpu(unsigned int irq, const struct cpumask *affinity) { - cpumask_t mask; int cpuid; - cpumask_copy(&mask, affinity); - if (cpumask_equal(&mask, cpu_online_mask)) { + if (cpumask_equal(affinity, cpu_online_mask)) { cpuid = map_to_cpu(irq); } else { - cpumask_t tmp; - - cpumask_and(&tmp, cpu_online_mask, &mask); - cpuid = cpumask_empty(&tmp) ? map_to_cpu(irq) : cpumask_first(&tmp); + cpuid = cpumask_first_and(affinity, cpu_online_mask); + cpuid = cpuid < nr_cpu_ids ? cpuid : map_to_cpu(irq); } return cpuid; @@ -240,13 +366,15 @@ static int irq_choose_cpu(unsigned int irq, const struct cpumask *affinity) static void sun4u_irq_enable(struct irq_data *data) { - struct irq_handler_data *handler_data = data->handler_data; + struct irq_handler_data *handler_data; + handler_data = irq_data_get_irq_handler_data(data); if (likely(handler_data)) { unsigned long cpuid, imap, val; unsigned int tid; - cpuid = irq_choose_cpu(data->irq, data->affinity); + cpuid = irq_choose_cpu(data->irq, + irq_data_get_affinity_mask(data)); imap = handler_data->imap; tid = sun4u_compute_tid(imap, cpuid); @@ -263,8 +391,9 @@ static void sun4u_irq_enable(struct irq_data *data) static int sun4u_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force) { - struct irq_handler_data *handler_data = data->handler_data; + struct irq_handler_data *handler_data; + handler_data = irq_data_get_irq_handler_data(data); if (likely(handler_data)) { unsigned long cpuid, imap, val; unsigned int tid; @@ -308,16 +437,18 @@ static void sun4u_irq_disable(struct irq_data *data) static void sun4u_irq_eoi(struct irq_data *data) { - struct irq_handler_data *handler_data = data->handler_data; + struct irq_handler_data *handler_data; + handler_data = irq_data_get_irq_handler_data(data); if (likely(handler_data)) upa_writeq(ICLR_IDLE, handler_data->iclr); } static void sun4v_irq_enable(struct irq_data *data) { - unsigned int ino = irq_table[data->irq].dev_ino; - unsigned long cpuid = irq_choose_cpu(data->irq, data->affinity); + unsigned long cpuid = irq_choose_cpu(data->irq, + irq_data_get_affinity_mask(data)); + unsigned int ino = irq_data_to_sysino(data); int err; err = sun4v_intr_settarget(ino, cpuid); @@ -337,8 +468,8 @@ static void sun4v_irq_enable(struct irq_data *data) static int sun4v_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force) { - unsigned int ino = irq_table[data->irq].dev_ino; unsigned long cpuid = irq_choose_cpu(data->irq, mask); + unsigned int ino = irq_data_to_sysino(data); int err; err = sun4v_intr_settarget(ino, cpuid); @@ -351,7 +482,7 @@ static int sun4v_set_affinity(struct irq_data *data, static void sun4v_irq_disable(struct irq_data *data) { - unsigned int ino = irq_table[data->irq].dev_ino; + unsigned int ino = irq_data_to_sysino(data); int err; err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED); @@ -362,7 +493,7 @@ static void sun4v_irq_disable(struct irq_data *data) static void sun4v_irq_eoi(struct irq_data *data) { - unsigned int ino = irq_table[data->irq].dev_ino; + unsigned int ino = irq_data_to_sysino(data); int err; err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE); @@ -373,13 +504,12 @@ static void sun4v_irq_eoi(struct irq_data *data) static void sun4v_virq_enable(struct irq_data *data) { - unsigned long cpuid, dev_handle, dev_ino; + unsigned long dev_handle = irq_data_to_handle(data); + unsigned long dev_ino = irq_data_to_ino(data); + unsigned long cpuid; int err; - cpuid = irq_choose_cpu(data->irq, data->affinity); - - dev_handle = irq_table[data->irq].dev_handle; - dev_ino = irq_table[data->irq].dev_ino; + cpuid = irq_choose_cpu(data->irq, irq_data_get_affinity_mask(data)); err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); if (err != HV_EOK) @@ -403,14 +533,13 @@ static void sun4v_virq_enable(struct irq_data *data) static int sun4v_virt_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force) { - unsigned long cpuid, dev_handle, dev_ino; + unsigned long dev_handle = irq_data_to_handle(data); + unsigned long dev_ino = irq_data_to_ino(data); + unsigned long cpuid; int err; cpuid = irq_choose_cpu(data->irq, mask); - dev_handle = irq_table[data->irq].dev_handle; - dev_ino = irq_table[data->irq].dev_ino; - err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); if (err != HV_EOK) printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): " @@ -422,11 +551,10 @@ static int sun4v_virt_set_affinity(struct irq_data *data, static void sun4v_virq_disable(struct irq_data *data) { - unsigned long dev_handle, dev_ino; + unsigned long dev_handle = irq_data_to_handle(data); + unsigned long dev_ino = irq_data_to_ino(data); int err; - dev_handle = irq_table[data->irq].dev_handle; - dev_ino = irq_table[data->irq].dev_ino; err = sun4v_vintr_set_valid(dev_handle, dev_ino, HV_INTR_DISABLED); @@ -438,12 +566,10 @@ static void sun4v_virq_disable(struct irq_data *data) static void sun4v_virq_eoi(struct irq_data *data) { - unsigned long dev_handle, dev_ino; + unsigned long dev_handle = irq_data_to_handle(data); + unsigned long dev_ino = irq_data_to_ino(data); int err; - dev_handle = irq_table[data->irq].dev_handle; - dev_ino = irq_table[data->irq].dev_ino; - err = sun4v_vintr_set_state(dev_handle, dev_ino, HV_INTR_STATE_IDLE); if (err != HV_EOK) @@ -479,31 +605,10 @@ static struct irq_chip sun4v_virq = { .flags = IRQCHIP_EOI_IF_HANDLED, }; -static void pre_flow_handler(struct irq_data *d) -{ - struct irq_handler_data *handler_data = irq_data_get_irq_handler_data(d); - unsigned int ino = irq_table[d->irq].dev_ino; - - handler_data->pre_handler(ino, handler_data->arg1, handler_data->arg2); -} - -void irq_install_pre_handler(int irq, - void (*func)(unsigned int, void *, void *), - void *arg1, void *arg2) -{ - struct irq_handler_data *handler_data = irq_get_handler_data(irq); - - handler_data->pre_handler = func; - handler_data->arg1 = arg1; - handler_data->arg2 = arg2; - - __irq_set_preflow_handler(irq, pre_flow_handler); -} - unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap) { - struct ino_bucket *bucket; struct irq_handler_data *handler_data; + struct ino_bucket *bucket; unsigned int irq; int ino; @@ -537,119 +642,166 @@ out: return irq; } -static unsigned int sun4v_build_common(unsigned long sysino, - struct irq_chip *chip) +static unsigned int sun4v_build_common(u32 devhandle, unsigned int devino, + void (*handler_data_init)(struct irq_handler_data *data, + u32 devhandle, unsigned int devino), + struct irq_chip *chip) { - struct ino_bucket *bucket; - struct irq_handler_data *handler_data; + struct irq_handler_data *data; unsigned int irq; - BUG_ON(tlb_type != hypervisor); + irq = irq_alloc(devhandle, devino); + if (!irq) + goto out; - bucket = &ivector_table[sysino]; - irq = bucket_get_irq(__pa(bucket)); - if (!irq) { - irq = irq_alloc(0, sysino); - bucket_set_irq(__pa(bucket), irq); - irq_set_chip_and_handler_name(irq, chip, handle_fasteoi_irq, - "IVEC"); + data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); + if (unlikely(!data)) { + pr_err("IRQ handler data allocation failed.\n"); + irq_free(irq); + irq = 0; + goto out; } - handler_data = irq_get_handler_data(irq); - if (unlikely(handler_data)) - goto out; + irq_set_handler_data(irq, data); + handler_data_init(data, devhandle, devino); + irq_set_chip_and_handler_name(irq, chip, handle_fasteoi_irq, "IVEC"); + data->imap = ~0UL; + data->iclr = ~0UL; +out: + return irq; +} - handler_data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); - if (unlikely(!handler_data)) { - prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n"); - prom_halt(); - } - irq_set_handler_data(irq, handler_data); +static unsigned long cookie_assign(unsigned int irq, u32 devhandle, + unsigned int devino) +{ + struct irq_handler_data *ihd = irq_get_handler_data(irq); + unsigned long hv_error, cookie; - /* Catch accidental accesses to these things. IMAP/ICLR handling - * is done by hypervisor calls on sun4v platforms, not by direct - * register accesses. + /* handler_irq needs to find the irq. cookie is seen signed in + * sun4v_dev_mondo and treated as a non ivector_table delivery. */ - handler_data->imap = ~0UL; - handler_data->iclr = ~0UL; + ihd->bucket.__irq = irq; + cookie = ~__pa(&ihd->bucket); -out: - return irq; + hv_error = sun4v_vintr_set_cookie(devhandle, devino, cookie); + if (hv_error) + pr_err("HV vintr set cookie failed = %ld\n", hv_error); + + return hv_error; } -unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino) +static void cookie_handler_data(struct irq_handler_data *data, + u32 devhandle, unsigned int devino) { - unsigned long sysino = sun4v_devino_to_sysino(devhandle, devino); + data->dev_handle = devhandle; + data->dev_ino = devino; +} + +static unsigned int cookie_build_irq(u32 devhandle, unsigned int devino, + struct irq_chip *chip) +{ + unsigned long hv_error; + unsigned int irq; + + irq = sun4v_build_common(devhandle, devino, cookie_handler_data, chip); + + hv_error = cookie_assign(irq, devhandle, devino); + if (hv_error) { + irq_free(irq); + irq = 0; + } - return sun4v_build_common(sysino, &sun4v_irq); + return irq; } -unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino) +static unsigned int sun4v_build_cookie(u32 devhandle, unsigned int devino) { - struct irq_handler_data *handler_data; - unsigned long hv_err, cookie; - struct ino_bucket *bucket; unsigned int irq; - bucket = kzalloc(sizeof(struct ino_bucket), GFP_ATOMIC); - if (unlikely(!bucket)) - return 0; + irq = cookie_exists(devhandle, devino); + if (irq) + goto out; - /* The only reference we store to the IRQ bucket is - * by physical address which kmemleak can't see, tell - * it that this object explicitly is not a leak and - * should be scanned. - */ - kmemleak_not_leak(bucket); + irq = cookie_build_irq(devhandle, devino, &sun4v_virq); - __flush_dcache_range((unsigned long) bucket, - ((unsigned long) bucket + - sizeof(struct ino_bucket))); +out: + return irq; +} - irq = irq_alloc(devhandle, devino); +static void sysino_set_bucket(unsigned int irq) +{ + struct irq_handler_data *ihd = irq_get_handler_data(irq); + struct ino_bucket *bucket; + unsigned long sysino; + + sysino = sun4v_devino_to_sysino(ihd->dev_handle, ihd->dev_ino); + BUG_ON(sysino >= nr_ivec); + bucket = &ivector_table[sysino]; bucket_set_irq(__pa(bucket), irq); +} - irq_set_chip_and_handler_name(irq, &sun4v_virq, handle_fasteoi_irq, - "IVEC"); +static void sysino_handler_data(struct irq_handler_data *data, + u32 devhandle, unsigned int devino) +{ + unsigned long sysino; - handler_data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); - if (unlikely(!handler_data)) - return 0; + sysino = sun4v_devino_to_sysino(devhandle, devino); + data->sysino = sysino; +} - /* In order to make the LDC channel startup sequence easier, - * especially wrt. locking, we do not let request_irq() enable - * the interrupt. - */ - irq_set_status_flags(irq, IRQ_NOAUTOEN); - irq_set_handler_data(irq, handler_data); +static unsigned int sysino_build_irq(u32 devhandle, unsigned int devino, + struct irq_chip *chip) +{ + unsigned int irq; - /* Catch accidental accesses to these things. IMAP/ICLR handling - * is done by hypervisor calls on sun4v platforms, not by direct - * register accesses. - */ - handler_data->imap = ~0UL; - handler_data->iclr = ~0UL; + irq = sun4v_build_common(devhandle, devino, sysino_handler_data, chip); + if (!irq) + goto out; - cookie = ~__pa(bucket); - hv_err = sun4v_vintr_set_cookie(devhandle, devino, cookie); - if (hv_err) { - prom_printf("IRQ: Fatal, cannot set cookie for [%x:%x] " - "err=%lu\n", devhandle, devino, hv_err); - prom_halt(); - } + sysino_set_bucket(irq); +out: + return irq; +} + +static int sun4v_build_sysino(u32 devhandle, unsigned int devino) +{ + int irq; + irq = sysino_exists(devhandle, devino); + if (irq) + goto out; + + irq = sysino_build_irq(devhandle, devino, &sun4v_irq); +out: return irq; } -void ack_bad_irq(unsigned int irq) +unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino) { - unsigned int ino = irq_table[irq].dev_ino; + unsigned int irq; - if (!ino) - ino = 0xdeadbeef; + if (sun4v_cookie_only_virqs()) + irq = sun4v_build_cookie(devhandle, devino); + else + irq = sun4v_build_sysino(devhandle, devino); - printk(KERN_CRIT "Unexpected IRQ from ino[%x] irq[%u]\n", - ino, irq); + return irq; +} + +unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino) +{ + int irq; + + irq = cookie_build_irq(devhandle, devino, &sun4v_virq); + if (!irq) + goto out; + + /* This is borrowed from the original function. + */ + irq_set_status_flags(irq, IRQ_NOAUTOEN); + +out: + return irq; } void *hardirq_stack[NR_CPUS]; @@ -698,31 +850,22 @@ void __irq_entry handler_irq(int pil, struct pt_regs *regs) set_irq_regs(old_regs); } -void do_softirq(void) +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK +void do_softirq_own_stack(void) { - unsigned long flags; - - if (in_interrupt()) - return; + void *orig_sp, *sp = softirq_stack[smp_processor_id()]; - local_irq_save(flags); + sp += THREAD_SIZE - 192 - STACK_BIAS; - if (local_softirq_pending()) { - void *orig_sp, *sp = softirq_stack[smp_processor_id()]; - - sp += THREAD_SIZE - 192 - STACK_BIAS; - - __asm__ __volatile__("mov %%sp, %0\n\t" - "mov %1, %%sp" - : "=&r" (orig_sp) - : "r" (sp)); - __do_softirq(); - __asm__ __volatile__("mov %0, %%sp" - : : "r" (orig_sp)); - } - - local_irq_restore(flags); + __asm__ __volatile__("mov %%sp, %0\n\t" + "mov %1, %%sp" + : "=&r" (orig_sp) + : "r" (sp)); + __do_softirq(); + __asm__ __volatile__("mov %0, %%sp" + : : "r" (orig_sp)); } +#endif #ifdef CONFIG_HOTPLUG_CPU void fixup_irqs(void) @@ -731,15 +874,18 @@ void fixup_irqs(void) for (irq = 0; irq < NR_IRQS; irq++) { struct irq_desc *desc = irq_to_desc(irq); - struct irq_data *data = irq_desc_get_irq_data(desc); + struct irq_data *data; unsigned long flags; + if (!desc) + continue; + data = irq_desc_get_irq_data(desc); raw_spin_lock_irqsave(&desc->lock, flags); if (desc->action && !irqd_is_per_cpu(data)) { if (data->chip->irq_set_affinity) data->chip->irq_set_affinity(data, - data->affinity, - false); + irq_data_get_affinity_mask(data), + false); } raw_spin_unlock_irqrestore(&desc->lock, flags); } @@ -767,7 +913,7 @@ static void map_prom_timers(void) dp = of_find_node_by_path("/"); dp = dp->child; while (dp) { - if (!strcmp(dp->name, "counter-timer")) + if (of_node_name_eq(dp, "counter-timer")) break; dp = dp->sibling; } @@ -829,13 +975,14 @@ void notrace init_irqwork_curcpu(void) * * On SMP this gets invoked from the CPU trampoline before * the cpu has fully taken over the trap table from OBP, - * and it's kernel stack + %g6 thread register state is + * and its kernel stack + %g6 thread register state is * not fully cooked yet. * * Therefore you cannot make any OBP calls, not even prom_printf, * from these two routines. */ -static void __cpuinit notrace register_one_mondo(unsigned long paddr, unsigned long type, unsigned long qmask) +static void notrace register_one_mondo(unsigned long paddr, unsigned long type, + unsigned long qmask) { unsigned long num_entries = (qmask + 1) / 64; unsigned long status; @@ -848,7 +995,7 @@ static void __cpuinit notrace register_one_mondo(unsigned long paddr, unsigned l } } -void __cpuinit notrace sun4v_register_mondo_queues(int this_cpu) +void notrace sun4v_register_mondo_queues(int this_cpu) { struct trap_per_cpu *tb = &trap_block[this_cpu]; @@ -872,7 +1019,7 @@ static void __init alloc_one_queue(unsigned long *pa_ptr, unsigned long qmask) unsigned long order = get_order(size); unsigned long p; - p = __get_free_pages(GFP_KERNEL, order); + p = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); if (!p) { prom_printf("SUN4V: Error, cannot allocate queue.\n"); prom_halt(); @@ -885,17 +1032,26 @@ static void __init init_cpu_send_mondo_info(struct trap_per_cpu *tb) { #ifdef CONFIG_SMP unsigned long page; + void *mondo, *p; - BUILD_BUG_ON((NR_CPUS * sizeof(u16)) > (PAGE_SIZE - 64)); + BUILD_BUG_ON((NR_CPUS * sizeof(u16)) > PAGE_SIZE); + + /* Make sure mondo block is 64byte aligned */ + p = kzalloc(127, GFP_KERNEL); + if (!p) { + prom_printf("SUN4V: Error, cannot allocate mondo block.\n"); + prom_halt(); + } + mondo = (void *)(((unsigned long)p + 63) & ~0x3f); + tb->cpu_mondo_block_pa = __pa(mondo); page = get_zeroed_page(GFP_KERNEL); if (!page) { - prom_printf("SUN4V: Error, cannot allocate cpu mondo page.\n"); + prom_printf("SUN4V: Error, cannot allocate cpu list page.\n"); prom_halt(); } - tb->cpu_mondo_block_pa = __pa(page); - tb->cpu_list_pa = __pa(page + 64); + tb->cpu_list_pa = __pa(page); #endif } @@ -932,16 +1088,22 @@ static struct irqaction timer_irq_action = { .name = "timer", }; -/* Only invoked on boot processor. */ -void __init init_IRQ(void) +static void __init irq_ivector_init(void) { - unsigned long size; + unsigned long size, order; + unsigned int ivecs; - map_prom_timers(); - kill_prom_timer(); + /* If we are doing cookie only VIRQs then we do not need the ivector + * table to process interrupts. + */ + if (sun4v_cookie_only_virqs()) + return; - size = sizeof(struct ino_bucket) * NUM_IVECS; - ivector_table = kzalloc(size, GFP_KERNEL); + ivecs = size_nr_ivec(); + size = sizeof(struct ino_bucket) * ivecs; + order = get_order(size); + ivector_table = (struct ino_bucket *) + __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); if (!ivector_table) { prom_printf("Fatal error, cannot allocate ivector_table\n"); prom_halt(); @@ -950,6 +1112,15 @@ void __init init_IRQ(void) ((unsigned long) ivector_table) + size); ivector_table_pa = __pa(ivector_table); +} + +/* Only invoked on boot processor.*/ +void __init init_IRQ(void) +{ + irq_init_hv(); + irq_ivector_init(); + map_prom_timers(); + kill_prom_timer(); if (tlb_type == hypervisor) sun4v_init_mondo_queues(); diff --git a/arch/sparc/kernel/itlb_miss.S b/arch/sparc/kernel/itlb_miss.S index 5a8377b54955..5a5d92482e8d 100644 --- a/arch/sparc/kernel/itlb_miss.S +++ b/arch/sparc/kernel/itlb_miss.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* ITLB ** ICACHE line 1: Context 0 check and TSB load */ ldxa [%g0] ASI_IMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer ldxa [%g0] ASI_IMMU, %g6 ! Get TAG TARGET diff --git a/arch/sparc/kernel/ivec.S b/arch/sparc/kernel/ivec.S index d29f92ebca5e..94ba2c3a29c1 100644 --- a/arch/sparc/kernel/ivec.S +++ b/arch/sparc/kernel/ivec.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* The registers for cross calls will be: * * DATA 0: [low 32-bits] Address of function to call, jmp to this diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c index 48565c11e82a..a4cfaeecaf5e 100644 --- a/arch/sparc/kernel/jump_label.c +++ b/arch/sparc/kernel/jump_label.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/kernel.h> #include <linux/types.h> #include <linux/mutex.h> @@ -8,34 +9,39 @@ #include <asm/cacheflush.h> -#ifdef HAVE_JUMP_LABEL - void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type) { - u32 val; u32 *insn = (u32 *) (unsigned long) entry->code; + u32 val; - if (type == JUMP_LABEL_ENABLE) { + if (type == JUMP_LABEL_JMP) { s32 off = (s32)entry->target - (s32)entry->code; + bool use_v9_branch = false; + + BUG_ON(off & 3); #ifdef CONFIG_SPARC64 - /* ba,pt %xcc, . + (off << 2) */ - val = 0x10680000 | ((u32) off >> 2); -#else - /* ba . + (off << 2) */ - val = 0x10800000 | ((u32) off >> 2); + if (off <= 0xfffff && off >= -0x100000) + use_v9_branch = true; #endif + if (use_v9_branch) { + /* WDISP19 - target is . + immed << 2 */ + /* ba,pt %xcc, . + off */ + val = 0x10680000 | (((u32) off >> 2) & 0x7ffff); + } else { + /* WDISP22 - target is . + immed << 2 */ + BUG_ON(off > 0x7fffff); + BUG_ON(off < -0x800000); + /* ba . + off */ + val = 0x10800000 | (((u32) off >> 2) & 0x3fffff); + } } else { val = 0x01000000; } - get_online_cpus(); mutex_lock(&text_mutex); *insn = val; flushi(insn); mutex_unlock(&text_mutex); - put_online_cpus(); } - -#endif diff --git a/arch/sparc/kernel/kernel.h b/arch/sparc/kernel/kernel.h index a702d9ab019c..8328a3b78a44 100644 --- a/arch/sparc/kernel/kernel.h +++ b/arch/sparc/kernel/kernel.h @@ -1,7 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __SPARC_KERNEL_H #define __SPARC_KERNEL_H #include <linux/interrupt.h> +#include <linux/ftrace.h> #include <asm/traps.h> #include <asm/head.h> @@ -12,74 +14,138 @@ extern const char *sparc_pmu_type; extern unsigned int fsr_storage; extern int ncpus_probed; +/* process{_32,_64}.c */ +asmlinkage long sparc_clone(struct pt_regs *regs); +asmlinkage long sparc_fork(struct pt_regs *regs); +asmlinkage long sparc_vfork(struct pt_regs *regs); + #ifdef CONFIG_SPARC64 /* setup_64.c */ struct seq_file; -extern void cpucap_info(struct seq_file *); +void cpucap_info(struct seq_file *); -static inline unsigned long kimage_addr_to_ra(const char *p) +static inline unsigned long kimage_addr_to_ra(const void *p) { unsigned long val = (unsigned long) p; return kern_base + (val - KERNBASE); } + +/* sys_sparc_64.c */ +asmlinkage long sys_kern_features(void); + +/* unaligned_64.c */ +asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn); +int handle_popc(u32 insn, struct pt_regs *regs); +void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr); +void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr); + +/* uprobes.c */ +asmlinkage void uprobe_trap(struct pt_regs *regs, + unsigned long trap_level); + +/* smp_64.c */ +void __irq_entry smp_call_function_client(int irq, struct pt_regs *regs); +void __irq_entry smp_call_function_single_client(int irq, struct pt_regs *regs); +void __irq_entry smp_penguin_jailcell(int irq, struct pt_regs *regs); +void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs); + +/* kgdb_64.c */ +void __irq_entry smp_kgdb_capture_client(int irq, struct pt_regs *regs); + +/* pci.c */ +#ifdef CONFIG_PCI +int ali_sound_dma_hack(struct device *dev, u64 device_mask); +#else +#define ali_sound_dma_hack(dev, mask) (0) +#endif + +/* signal32.c */ +void do_sigreturn32(struct pt_regs *regs); +asmlinkage void do_rt_sigreturn32(struct pt_regs *regs); +void do_signal32(struct pt_regs * regs); +asmlinkage int do_sys32_sigstack(u32 u_ssptr, u32 u_ossptr, unsigned long sp); + +/* time_64.c */ +void __init time_init_early(void); + +/* compat_audit.c */ +extern unsigned int sparc32_dir_class[]; +extern unsigned int sparc32_chattr_class[]; +extern unsigned int sparc32_write_class[]; +extern unsigned int sparc32_read_class[]; +extern unsigned int sparc32_signal_class[]; +int sparc32_classify_syscall(unsigned int syscall); #endif #ifdef CONFIG_SPARC32 /* setup_32.c */ +struct linux_romvec; void sparc32_start_kernel(struct linux_romvec *rp); /* cpu.c */ -extern void cpu_probe(void); +void cpu_probe(void); /* traps_32.c */ -extern void handle_hw_divzero(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); +void handle_hw_divzero(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); /* irq_32.c */ extern struct irqaction static_irqaction[]; extern int static_irq_count; extern spinlock_t irq_action_lock; -extern void unexpected_irq(int irq, void *dev_id, struct pt_regs * regs); -extern void init_IRQ(void); +void unexpected_irq(int irq, void *dev_id, struct pt_regs * regs); /* sun4m_irq.c */ -extern void sun4m_init_IRQ(void); -extern void sun4m_unmask_profile_irq(void); -extern void sun4m_clear_profile_irq(int cpu); +void sun4m_init_IRQ(void); +void sun4m_unmask_profile_irq(void); +void sun4m_clear_profile_irq(int cpu); /* sun4m_smp.c */ void sun4m_cpu_pre_starting(void *arg); void sun4m_cpu_pre_online(void *arg); +void __init smp4m_boot_cpus(void); +int smp4m_boot_one_cpu(int i, struct task_struct *idle); +void __init smp4m_smp_done(void); +void smp4m_cross_call_irq(void); +void smp4m_percpu_timer_interrupt(struct pt_regs *regs); /* sun4d_irq.c */ extern spinlock_t sun4d_imsk_lock; -extern void sun4d_init_IRQ(void); -extern int sun4d_request_irq(unsigned int irq, - irq_handler_t handler, - unsigned long irqflags, - const char *devname, void *dev_id); -extern int show_sun4d_interrupts(struct seq_file *, void *); -extern void sun4d_distribute_irqs(void); -extern void sun4d_free_irq(unsigned int irq, void *dev_id); +void sun4d_init_IRQ(void); +int sun4d_request_irq(unsigned int irq, + irq_handler_t handler, + unsigned long irqflags, + const char *devname, void *dev_id); +int show_sun4d_interrupts(struct seq_file *, void *); +void sun4d_distribute_irqs(void); +void sun4d_free_irq(unsigned int irq, void *dev_id); /* sun4d_smp.c */ void sun4d_cpu_pre_starting(void *arg); void sun4d_cpu_pre_online(void *arg); +void __init smp4d_boot_cpus(void); +int smp4d_boot_one_cpu(int i, struct task_struct *idle); +void __init smp4d_smp_done(void); +void smp4d_cross_call_irq(void); +void smp4d_percpu_timer_interrupt(struct pt_regs *regs); /* leon_smp.c */ void leon_cpu_pre_starting(void *arg); void leon_cpu_pre_online(void *arg); +void leonsmp_ipi_interrupt(void); +void leon_cross_call_irq(void); /* head_32.S */ extern unsigned int t_nmi[]; extern unsigned int linux_trap_ipi15_sun4d[]; extern unsigned int linux_trap_ipi15_sun4m[]; -extern struct tt_entry trapbase_cpu1; -extern struct tt_entry trapbase_cpu2; -extern struct tt_entry trapbase_cpu3; +extern struct tt_entry trapbase[]; +extern struct tt_entry trapbase_cpu1[]; +extern struct tt_entry trapbase_cpu2[]; +extern struct tt_entry trapbase_cpu3[]; extern char cputypval[]; @@ -89,12 +155,42 @@ extern unsigned int real_irq_entry[]; extern unsigned int smp4d_ticker[]; extern unsigned int patchme_maybe_smp_msg[]; -extern void floppy_hardint(void); +void floppy_hardint(void); /* trampoline_32.S */ extern unsigned long sun4m_cpu_startup; extern unsigned long sun4d_cpu_startup; +/* signal_32.c */ +asmlinkage void do_sigreturn(struct pt_regs *regs); +asmlinkage void do_rt_sigreturn(struct pt_regs *regs); +void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, + unsigned long thread_info_flags); +asmlinkage int do_sys_sigstack(struct sigstack __user *ssptr, + struct sigstack __user *ossptr, + unsigned long sp); + +/* ptrace_32.c */ +asmlinkage int syscall_trace(struct pt_regs *regs, int syscall_exit_p); + +/* unaligned_32.c */ +asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn); +asmlinkage void user_unaligned_trap(struct pt_regs *regs, unsigned int insn); + +/* windows.c */ +void try_to_clear_window_buffer(struct pt_regs *regs, int who); + +/* auxio_32.c */ +void __init auxio_probe(void); +void __init auxio_power_probe(void); + +/* pcic.c */ +extern void __iomem *pcic_regs; +void pcic_nmi(unsigned int pend, struct pt_regs *regs); + +/* time_32.c */ +void __init time_init(void); + #else /* CONFIG_SPARC32 */ #endif /* CONFIG_SPARC32 */ #endif /* !(__SPARC_KERNEL_H) */ diff --git a/arch/sparc/kernel/kgdb_32.c b/arch/sparc/kernel/kgdb_32.c index dcf210811af4..3b2c673ec627 100644 --- a/arch/sparc/kernel/kgdb_32.c +++ b/arch/sparc/kernel/kgdb_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* kgdb.c: KGDB support for 32-bit sparc. * * Copyright (C) 2008 David S. Miller <davem@davemloft.net> @@ -12,7 +13,8 @@ #include <asm/irq.h> #include <asm/cacheflush.h> -extern unsigned long trapbase; +#include "kernel.h" +#include "entry.h" void pt_regs_to_gdb_regs(unsigned long *gdb_regs, struct pt_regs *regs) { @@ -35,7 +37,7 @@ void pt_regs_to_gdb_regs(unsigned long *gdb_regs, struct pt_regs *regs) gdb_regs[GDB_Y] = regs->y; gdb_regs[GDB_PSR] = regs->psr; gdb_regs[GDB_WIM] = 0; - gdb_regs[GDB_TBR] = (unsigned long) &trapbase; + gdb_regs[GDB_TBR] = (unsigned long) &trapbase[0]; gdb_regs[GDB_PC] = regs->pc; gdb_regs[GDB_NPC] = regs->npc; gdb_regs[GDB_FSR] = 0; @@ -70,7 +72,7 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) gdb_regs[GDB_PSR] = t->kpsr; gdb_regs[GDB_WIM] = t->kwim; - gdb_regs[GDB_TBR] = (unsigned long) &trapbase; + gdb_regs[GDB_TBR] = (unsigned long) &trapbase[0]; gdb_regs[GDB_PC] = t->kpc; gdb_regs[GDB_NPC] = t->kpc + 4; gdb_regs[GDB_FSR] = 0; @@ -120,7 +122,7 @@ int kgdb_arch_handle_exception(int e_vector, int signo, int err_code, linux_regs->pc = addr; linux_regs->npc = addr + 4; } - /* fallthru */ + fallthrough; case 'D': case 'k': @@ -133,21 +135,19 @@ int kgdb_arch_handle_exception(int e_vector, int signo, int err_code, return -1; } -extern void do_hw_interrupt(struct pt_regs *regs, unsigned long type); - -asmlinkage void kgdb_trap(struct pt_regs *regs) +asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs) { unsigned long flags; if (user_mode(regs)) { - do_hw_interrupt(regs, 0xfd); + do_hw_interrupt(regs, trap_level); return; } flushw_all(); local_irq_save(flags); - kgdb_handle_exception(0x172, SIGTRAP, 0, regs); + kgdb_handle_exception(trap_level, SIGTRAP, 0, regs); local_irq_restore(flags); } @@ -166,7 +166,7 @@ void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long ip) regs->npc = regs->pc + 4; } -struct kgdb_arch arch_kgdb_ops = { +const struct kgdb_arch arch_kgdb_ops = { /* Breakpoint instruction: ta 0x7d */ .gdb_bpt_instr = { 0x91, 0xd0, 0x20, 0x7d }, }; diff --git a/arch/sparc/kernel/kgdb_64.c b/arch/sparc/kernel/kgdb_64.c index c8759550799f..177746ae2c81 100644 --- a/arch/sparc/kernel/kgdb_64.c +++ b/arch/sparc/kernel/kgdb_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* kgdb.c: KGDB support for 64-bit sparc. * * Copyright (C) 2008 David S. Miller <davem@davemloft.net> @@ -6,12 +7,15 @@ #include <linux/kgdb.h> #include <linux/kdebug.h> #include <linux/ftrace.h> +#include <linux/context_tracking.h> #include <asm/cacheflush.h> #include <asm/kdebug.h> #include <asm/ptrace.h> #include <asm/irq.h> +#include "kernel.h" + void pt_regs_to_gdb_regs(unsigned long *gdb_regs, struct pt_regs *regs) { struct reg_window *win; @@ -42,7 +46,7 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) { struct thread_info *t = task_thread_info(p); extern unsigned int switch_to_pc; - extern unsigned int ret_from_syscall; + extern unsigned int ret_from_fork; struct reg_window *win; unsigned long pc, cwp; int i; @@ -66,7 +70,7 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) gdb_regs[i] = 0; if (t->new_child) - pc = (unsigned long) &ret_from_syscall; + pc = (unsigned long) &ret_from_fork; else pc = (unsigned long) &switch_to_pc; @@ -144,7 +148,7 @@ int kgdb_arch_handle_exception(int e_vector, int signo, int err_code, linux_regs->tpc = addr; linux_regs->tnpc = addr + 4; } - /* fallthru */ + fallthrough; case 'D': case 'k': @@ -159,11 +163,12 @@ int kgdb_arch_handle_exception(int e_vector, int signo, int err_code, asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); unsigned long flags; if (user_mode(regs)) { bad_trap(regs, trap_level); - return; + goto out; } flushw_all(); @@ -171,6 +176,8 @@ asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs) local_irq_save(flags); kgdb_handle_exception(0x172, SIGTRAP, 0, regs); local_irq_restore(flags); +out: + exception_exit(prev_state); } int kgdb_arch_init(void) @@ -188,7 +195,7 @@ void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long ip) regs->tnpc = regs->tpc + 4; } -struct kgdb_arch arch_kgdb_ops = { +const struct kgdb_arch arch_kgdb_ops = { /* Breakpoint instruction: ta 0x72 */ .gdb_bpt_instr = { 0x91, 0xd0, 0x20, 0x72 }, }; diff --git a/arch/sparc/kernel/kprobes.c b/arch/sparc/kernel/kprobes.c index e72212148d2a..191bbaca9921 100644 --- a/arch/sparc/kernel/kprobes.c +++ b/arch/sparc/kernel/kprobes.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* arch/sparc64/kernel/kprobes.c * * Copyright (C) 2004 David S. Miller <davem@davemloft.net> @@ -5,12 +6,13 @@ #include <linux/kernel.h> #include <linux/kprobes.h> -#include <linux/module.h> +#include <linux/extable.h> #include <linux/kdebug.h> #include <linux/slab.h> +#include <linux/context_tracking.h> #include <asm/signal.h> #include <asm/cacheflush.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> /* We do not have hardware single-stepping on sparc64. * So we implement software single-stepping with breakpoint @@ -82,7 +84,7 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) { - __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp; + __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status; kcb->kprobe_orig_tnpc = kcb->prev_kprobe.orig_tnpc; kcb->kprobe_orig_tstate_pil = kcb->prev_kprobe.orig_tstate_pil; @@ -91,7 +93,7 @@ static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb) { - __get_cpu_var(current_kprobe) = p; + __this_cpu_write(current_kprobe, p); kcb->kprobe_orig_tnpc = regs->tnpc; kcb->kprobe_orig_tstate_pil = (regs->tstate & TSTATE_PIL); } @@ -145,18 +147,12 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) kcb->kprobe_status = KPROBE_REENTER; prepare_singlestep(p, regs, kcb); return 1; - } else { - if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { + } else if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { /* The breakpoint instruction was removed by * another cpu right after we hit, no further * handling of this interrupt is appropriate */ - ret = 1; - goto no_kprobe; - } - p = __get_cpu_var(current_kprobe); - if (p->break_handler && p->break_handler(p, regs)) - goto ss_probe; + ret = 1; } goto no_kprobe; } @@ -179,10 +175,12 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) set_current_kprobe(p, regs, kcb); kcb->kprobe_status = KPROBE_HIT_ACTIVE; - if (p->pre_handler && p->pre_handler(p, regs)) + if (p->pre_handler && p->pre_handler(p, regs)) { + reset_current_kprobe(); + preempt_enable_no_resched(); return 1; + } -ss_probe: prepare_singlestep(p, regs, kcb); kcb->kprobe_status = KPROBE_HIT_SS; return 1; @@ -232,7 +230,7 @@ static unsigned long __kprobes relbranch_fixup(u32 insn, struct kprobe *p, return regs->tnpc; } -/* If INSN is an instruction which writes it's PC location +/* If INSN is an instruction which writes its PC location * into a destination register, fix that up. */ static void __kprobes retpc_fixup(struct pt_regs *regs, u32 insn, @@ -348,23 +346,6 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) case KPROBE_HIT_ACTIVE: case KPROBE_HIT_SSDONE: /* - * We increment the nmissed count for accounting, - * we can also use npre/npostfault count for accouting - * these specific fault cases. - */ - kprobes_inc_nmissed_count(cur); - - /* - * We come here because instructions in the pre/post - * handler caused the page_fault, this could happen - * if handler tries to access user space by - * copy_from_user(), get_user() etc. Let the - * user-specified handler try to fix it first. - */ - if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) - return 1; - - /* * In case the user-specified fault handler returned * zero, try to fix up. */ @@ -418,12 +399,14 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); + BUG_ON(trap_level != 0x170 && trap_level != 0x171); if (user_mode(regs)) { local_irq_enable(); bad_trap(regs, trap_level); - return; + goto out; } /* trap_level == 0x170 --> ta 0x70 @@ -433,53 +416,8 @@ asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, (trap_level == 0x170) ? "debug" : "debug_2", regs, 0, trap_level, SIGTRAP) != NOTIFY_STOP) bad_trap(regs, trap_level); -} - -/* Jprobes support. */ -int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct jprobe *jp = container_of(p, struct jprobe, kp); - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - memcpy(&(kcb->jprobe_saved_regs), regs, sizeof(*regs)); - - regs->tpc = (unsigned long) jp->entry; - regs->tnpc = ((unsigned long) jp->entry) + 0x4UL; - regs->tstate |= TSTATE_PIL; - - return 1; -} - -void __kprobes jprobe_return(void) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - register unsigned long orig_fp asm("g1"); - - orig_fp = kcb->jprobe_saved_regs.u_regs[UREG_FP]; - __asm__ __volatile__("\n" -"1: cmp %%sp, %0\n\t" - "blu,a,pt %%xcc, 1b\n\t" - " restore\n\t" - ".globl jprobe_return_trap_instruction\n" -"jprobe_return_trap_instruction:\n\t" - "ta 0x70" - : /* no outputs */ - : "r" (orig_fp)); -} - -extern void jprobe_return_trap_instruction(void); - -int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) -{ - u32 *addr = (u32 *) regs->tpc; - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - if (addr == (u32 *) jprobe_return_trap_instruction) { - memcpy(regs, &(kcb->jprobe_saved_regs), sizeof(*regs)); - preempt_enable_no_resched(); - return 1; - } - return 0; +out: + exception_exit(prev_state); } /* The value stored in the return address register is actually 2 @@ -498,71 +436,25 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) { ri->ret_addr = (kprobe_opcode_t *)(regs->u_regs[UREG_RETPC] + 8); + ri->fp = NULL; /* Replace the return addr with trampoline addr */ regs->u_regs[UREG_RETPC] = - ((unsigned long)kretprobe_trampoline) - 8; + ((unsigned long)__kretprobe_trampoline) - 8; } /* * Called when the probe at kretprobe trampoline is hit */ -int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) +static int __kprobes trampoline_probe_handler(struct kprobe *p, + struct pt_regs *regs) { - struct kretprobe_instance *ri = NULL; - struct hlist_head *head, empty_rp; - struct hlist_node *tmp; - unsigned long flags, orig_ret_address = 0; - unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline; - - INIT_HLIST_HEAD(&empty_rp); - kretprobe_hash_lock(current, &head, &flags); + unsigned long orig_ret_address = 0; - /* - * It is possible to have multiple instances associated with a given - * task either because an multiple functions in the call path - * have a return probe installed on them, and/or more than one return - * return probe was registered for a target function. - * - * We can handle this because: - * - instances are always inserted at the head of the list - * - when multiple return probes are registered for the same - * function, the first instance's ret_addr will point to the - * real return address, and all the rest will point to - * kretprobe_trampoline - */ - hlist_for_each_entry_safe(ri, tmp, head, hlist) { - if (ri->task != current) - /* another task is sharing our hash bucket */ - continue; - - if (ri->rp && ri->rp->handler) - ri->rp->handler(ri, regs); - - orig_ret_address = (unsigned long)ri->ret_addr; - recycle_rp_inst(ri, &empty_rp); - - if (orig_ret_address != trampoline_address) - /* - * This is the real return address. Any other - * instances associated with this task are for - * other calls deeper on the call stack - */ - break; - } - - kretprobe_assert(ri, orig_ret_address, trampoline_address); + orig_ret_address = __kretprobe_trampoline_handler(regs, NULL); regs->tpc = orig_ret_address; regs->tnpc = orig_ret_address + 4; - reset_current_kprobe(); - kretprobe_hash_unlock(current, &flags); - preempt_enable_no_resched(); - - hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { - hlist_del(&ri->hlist); - kfree(ri); - } /* * By returning a non-zero value, we are telling * kprobe_handler() that we don't want the post_handler @@ -571,15 +463,15 @@ int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) return 1; } -void kretprobe_trampoline_holder(void) +static void __used kretprobe_trampoline_holder(void) { - asm volatile(".global kretprobe_trampoline\n" - "kretprobe_trampoline:\n" + asm volatile(".global __kretprobe_trampoline\n" + "__kretprobe_trampoline:\n" "\tnop\n" "\tnop\n"); } static struct kprobe trampoline_p = { - .addr = (kprobe_opcode_t *) &kretprobe_trampoline, + .addr = (kprobe_opcode_t *) &__kretprobe_trampoline, .pre_handler = trampoline_probe_handler }; @@ -590,7 +482,7 @@ int __init arch_init_kprobes(void) int __kprobes arch_trampoline_kprobe(struct kprobe *p) { - if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline) + if (p->addr == (kprobe_opcode_t *)&__kretprobe_trampoline) return 1; return 0; diff --git a/arch/sparc/kernel/kstack.h b/arch/sparc/kernel/kstack.h index 53dfb92e09fb..b3c5e8f2443a 100644 --- a/arch/sparc/kernel/kstack.h +++ b/arch/sparc/kernel/kstack.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _KSTACK_H #define _KSTACK_H diff --git a/arch/sparc/kernel/ktlb.S b/arch/sparc/kernel/ktlb.S index 0746e5e32b37..6bfaf73ce8a0 100644 --- a/arch/sparc/kernel/ktlb.S +++ b/arch/sparc/kernel/ktlb.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling. * * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net> @@ -6,10 +7,10 @@ * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ +#include <linux/pgtable.h> #include <asm/head.h> #include <asm/asi.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/tsb.h> .text @@ -20,16 +21,19 @@ kvmap_itlb: mov TLB_TAG_ACCESS, %g4 ldxa [%g4] ASI_IMMU, %g4 + /* The kernel executes in context zero, therefore we do not + * need to clear the context ID bits out of %g4 here. + */ + /* sun4v_itlb_miss branches here with the missing virtual * address already loaded into %g4 */ kvmap_itlb_4v: -kvmap_itlb_nonlinear: /* Catch kernel NULL pointer calls. */ sethi %hi(PAGE_SIZE), %g5 cmp %g4, %g5 - bleu,pn %xcc, kvmap_dtlb_longpath + blu,pn %xcc, kvmap_itlb_longpath nop KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load) @@ -48,14 +52,6 @@ kvmap_itlb_vmalloc_addr: KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath) TSB_LOCK_TAG(%g1, %g2, %g7) - - /* Load and check PTE. */ - ldxa [%g5] ASI_PHYS_USE_EC, %g5 - mov 1, %g7 - sllx %g7, TSB_TAG_INVALID_BIT, %g7 - brgez,a,pn %g5, kvmap_itlb_longpath - TSB_STORE(%g1, %g7) - TSB_WRITE(%g1, %g5, %g6) /* fallthrough to TLB load */ @@ -119,6 +115,12 @@ kvmap_dtlb_obp: ba,pt %xcc, kvmap_dtlb_load nop +kvmap_linear_early: + sethi %hi(kern_linear_pte_xor), %g7 + ldx [%g7 + %lo(kern_linear_pte_xor)], %g2 + ba,pt %xcc, kvmap_dtlb_tsb4m_load + xor %g2, %g4, %g5 + .align 32 kvmap_dtlb_tsb4m_load: TSB_LOCK_TAG(%g1, %g2, %g7) @@ -131,6 +133,10 @@ kvmap_dtlb: mov TLB_TAG_ACCESS, %g4 ldxa [%g4] ASI_DMMU, %g4 + /* The kernel executes in context zero, therefore we do not + * need to clear the context ID bits out of %g4 here. + */ + /* sun4v_dtlb_miss branches here with the missing virtual * address already loaded into %g4 */ @@ -147,85 +153,17 @@ kvmap_dtlb_4v: /* Correct TAG_TARGET is already in %g6, check 4mb TSB. */ KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) #endif - /* TSB entry address left in %g1, lookup linear PTE. - * Must preserve %g1 and %g6 (TAG). - */ -kvmap_dtlb_tsb4m_miss: - /* Clear the PAGE_OFFSET top virtual bits, shift - * down to get PFN, and make sure PFN is in range. - */ - sllx %g4, 21, %g5 - - /* Check to see if we know about valid memory at the 4MB - * chunk this physical address will reside within. - */ - srlx %g5, 21 + 41, %g2 - brnz,pn %g2, kvmap_dtlb_longpath - nop - - /* This unconditional branch and delay-slot nop gets patched - * by the sethi sequence once the bitmap is properly setup. + /* Linear mapping TSB lookup failed. Fallthrough to kernel + * page table based lookup. */ - .globl valid_addr_bitmap_insn -valid_addr_bitmap_insn: - ba,pt %xcc, 2f - nop - .subsection 2 - .globl valid_addr_bitmap_patch -valid_addr_bitmap_patch: - sethi %hi(sparc64_valid_addr_bitmap), %g7 - or %g7, %lo(sparc64_valid_addr_bitmap), %g7 - .previous - - srlx %g5, 21 + 22, %g2 - srlx %g2, 6, %g5 - and %g2, 63, %g2 - sllx %g5, 3, %g5 - ldx [%g7 + %g5], %g5 - mov 1, %g7 - sllx %g7, %g2, %g7 - andcc %g5, %g7, %g0 - be,pn %xcc, kvmap_dtlb_longpath - -2: sethi %hi(kpte_linear_bitmap), %g2 - - /* Get the 256MB physical address index. */ - sllx %g4, 21, %g5 - or %g2, %lo(kpte_linear_bitmap), %g2 - srlx %g5, 21 + 28, %g5 - and %g5, (32 - 1), %g7 - - /* Divide by 32 to get the offset into the bitmask. */ - srlx %g5, 5, %g5 - add %g7, %g7, %g7 - sllx %g5, 3, %g5 - - /* kern_linear_pte_xor[(mask >> shift) & 3)] */ - ldx [%g2 + %g5], %g2 - srlx %g2, %g7, %g7 - sethi %hi(kern_linear_pte_xor), %g5 - and %g7, 3, %g7 - or %g5, %lo(kern_linear_pte_xor), %g5 - sllx %g7, 3, %g7 - ldx [%g5 + %g7], %g2 - .globl kvmap_linear_patch kvmap_linear_patch: - ba,pt %xcc, kvmap_dtlb_tsb4m_load - xor %g2, %g4, %g5 + ba,a,pt %xcc, kvmap_linear_early kvmap_dtlb_vmalloc_addr: KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) TSB_LOCK_TAG(%g1, %g2, %g7) - - /* Load and check PTE. */ - ldxa [%g5] ASI_PHYS_USE_EC, %g5 - mov 1, %g7 - sllx %g7, TSB_TAG_INVALID_BIT, %g7 - brgez,a,pn %g5, kvmap_dtlb_longpath - TSB_STORE(%g1, %g7) - TSB_WRITE(%g1, %g5, %g6) /* fallthrough to TLB load */ @@ -257,13 +195,8 @@ kvmap_dtlb_load: #ifdef CONFIG_SPARSEMEM_VMEMMAP kvmap_vmemmap: - sub %g4, %g5, %g5 - srlx %g5, 22, %g5 - sethi %hi(vmemmap_table), %g1 - sllx %g5, 3, %g5 - or %g1, %lo(vmemmap_table), %g1 - ba,pt %xcc, kvmap_dtlb_load - ldx [%g1 + %g5], %g5 + KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) + ba,a,pt %xcc, kvmap_dtlb_load #endif kvmap_dtlb_nonlinear: @@ -275,8 +208,8 @@ kvmap_dtlb_nonlinear: #ifdef CONFIG_SPARSEMEM_VMEMMAP /* Do not use the TSB for vmemmap. */ - mov (VMEMMAP_BASE >> 40), %g5 - sllx %g5, 40, %g5 + sethi %hi(VMEMMAP_BASE), %g5 + ldx [%g5 + %lo(VMEMMAP_BASE)], %g5 cmp %g4,%g5 bgeu,pn %xcc, kvmap_vmemmap nop @@ -288,8 +221,8 @@ kvmap_dtlb_tsbmiss: sethi %hi(MODULES_VADDR), %g5 cmp %g4, %g5 blu,pn %xcc, kvmap_dtlb_longpath - mov (VMALLOC_END >> 40), %g5 - sllx %g5, 40, %g5 + sethi %hi(VMALLOC_END), %g5 + ldx [%g5 + %lo(VMALLOC_END)], %g5 cmp %g4, %g5 bgeu,pn %xcc, kvmap_dtlb_longpath nop @@ -327,6 +260,10 @@ kvmap_dtlb_longpath: nop .previous + /* The kernel executes in context zero, therefore we do not + * need to clear the context ID bits out of %g5 here. + */ + be,pt %xcc, sparc64_realfault_common mov FAULT_CODE_DTLB, %g4 ba,pt %xcc, winfix_trampoline diff --git a/arch/sparc/kernel/ldc.c b/arch/sparc/kernel/ldc.c index 54df554b82d9..7f3cdb6f644d 100644 --- a/arch/sparc/kernel/ldc.c +++ b/arch/sparc/kernel/ldc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* ldc.c: Logical Domain Channel link-layer protocol driver. * * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> @@ -15,6 +16,7 @@ #include <linux/list.h> #include <linux/init.h> #include <linux/bitmap.h> +#include <asm/iommu-common.h> #include <asm/hypervisor.h> #include <asm/iommu.h> @@ -27,9 +29,12 @@ #define DRV_MODULE_VERSION "1.1" #define DRV_MODULE_RELDATE "July 22, 2008" +#define COOKIE_PGSZ_CODE 0xf000000000000000ULL +#define COOKIE_PGSZ_CODE_SHIFT 60ULL + + static char version[] = DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; -#define LDC_PACKET_SIZE 64 /* Packet header layout for unreliable and reliable mode frames. * When in RAW mode, packets are simply straight 64-byte payloads @@ -98,10 +103,10 @@ static const struct ldc_mode_ops stream_ops; int ldom_domaining_enabled; struct ldc_iommu { - /* Protects arena alloc/free. */ + /* Protects ldc_unmap. */ spinlock_t lock; - struct iommu_arena arena; struct ldc_mtable_entry *page_table; + struct iommu_map_table iommu_map_table; }; struct ldc_channel { @@ -173,6 +178,8 @@ do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \ printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \ } while (0) +#define LDC_ABORT(lp) ldc_abort((lp), __func__) + static const char *state_to_str(u8 state) { switch (state) { @@ -191,15 +198,6 @@ static const char *state_to_str(u8 state) } } -static void ldc_set_state(struct ldc_channel *lp, u8 state) -{ - ldcdbg(STATE, "STATE (%s) --> (%s)\n", - state_to_str(lp->state), - state_to_str(state)); - - lp->state = state; -} - static unsigned long __advance(unsigned long off, unsigned long num_entries) { off += LDC_PACKET_SIZE; @@ -511,11 +509,12 @@ static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt) return err; } -static int ldc_abort(struct ldc_channel *lp) +static int ldc_abort(struct ldc_channel *lp, const char *msg) { unsigned long hv_err; - ldcdbg(STATE, "ABORT\n"); + ldcdbg(STATE, "ABORT[%s]\n", msg); + ldc_print(lp); /* We report but do not act upon the hypervisor errors because * there really isn't much we can do if they fail at this point. @@ -600,7 +599,7 @@ static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp) } } if (err) - return ldc_abort(lp); + return LDC_ABORT(lp); return 0; } @@ -613,13 +612,13 @@ static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp) if (lp->hs_state == LDC_HS_GOTVERS) { if (lp->ver.major != vp->major || lp->ver.minor != vp->minor) - return ldc_abort(lp); + return LDC_ABORT(lp); } else { lp->ver = *vp; lp->hs_state = LDC_HS_GOTVERS; } if (send_rts(lp)) - return ldc_abort(lp); + return LDC_ABORT(lp); return 0; } @@ -630,17 +629,17 @@ static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp) unsigned long new_tail; if (vp->major == 0 && vp->minor == 0) - return ldc_abort(lp); + return LDC_ABORT(lp); vap = find_by_major(vp->major); if (!vap) - return ldc_abort(lp); + return LDC_ABORT(lp); p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, vap, sizeof(*vap), &new_tail); if (!p) - return ldc_abort(lp); + return LDC_ABORT(lp); return send_tx_packet(lp, p, new_tail); } @@ -663,7 +662,7 @@ static int process_version(struct ldc_channel *lp, return process_ver_nack(lp, vp); default: - return ldc_abort(lp); + return LDC_ABORT(lp); } } @@ -676,13 +675,13 @@ static int process_rts(struct ldc_channel *lp, if (p->stype != LDC_INFO || lp->hs_state != LDC_HS_GOTVERS || p->env != lp->cfg.mode) - return ldc_abort(lp); + return LDC_ABORT(lp); lp->snd_nxt = p->seqid; lp->rcv_nxt = p->seqid; lp->hs_state = LDC_HS_SENTRTR; if (send_rtr(lp)) - return ldc_abort(lp); + return LDC_ABORT(lp); return 0; } @@ -695,7 +694,7 @@ static int process_rtr(struct ldc_channel *lp, if (p->stype != LDC_INFO || p->env != lp->cfg.mode) - return ldc_abort(lp); + return LDC_ABORT(lp); lp->snd_nxt = p->seqid; lp->hs_state = LDC_HS_COMPLETE; @@ -718,7 +717,7 @@ static int process_rdx(struct ldc_channel *lp, if (p->stype != LDC_INFO || !(rx_seq_ok(lp, p->seqid))) - return ldc_abort(lp); + return LDC_ABORT(lp); lp->rcv_nxt = p->seqid; @@ -745,14 +744,14 @@ static int process_control_frame(struct ldc_channel *lp, return process_rdx(lp, p); default: - return ldc_abort(lp); + return LDC_ABORT(lp); } } static int process_error_frame(struct ldc_channel *lp, struct ldc_packet *p) { - return ldc_abort(lp); + return LDC_ABORT(lp); } static int process_data_ack(struct ldc_channel *lp, @@ -771,7 +770,7 @@ static int process_data_ack(struct ldc_channel *lp, return 0; } if (head == lp->tx_tail) - return ldc_abort(lp); + return LDC_ABORT(lp); } return 0; @@ -815,16 +814,21 @@ static irqreturn_t ldc_rx(int irq, void *dev_id) lp->hs_state = LDC_HS_COMPLETE; ldc_set_state(lp, LDC_STATE_CONNECTED); - event_mask |= LDC_EVENT_UP; - - orig_state = lp->chan_state; + /* + * Generate an LDC_EVENT_UP event if the channel + * was not already up. + */ + if (orig_state != LDC_CHANNEL_UP) { + event_mask |= LDC_EVENT_UP; + orig_state = lp->chan_state; + } } /* If we are in reset state, flush the RX queue and ignore * everything. */ if (lp->flags & LDC_FLAG_RESET) { - (void) __set_rx_head(lp, lp->rx_tail); + (void) ldc_rx_reset(lp); goto out; } @@ -875,7 +879,7 @@ handshake_complete: break; default: - err = ldc_abort(lp); + err = LDC_ABORT(lp); break; } @@ -890,7 +894,7 @@ handshake_complete: err = __set_rx_head(lp, new); if (err < 0) { - (void) ldc_abort(lp); + (void) LDC_ABORT(lp); break; } if (lp->hs_state == LDC_HS_COMPLETE) @@ -931,7 +935,14 @@ static irqreturn_t ldc_tx(int irq, void *dev_id) lp->hs_state = LDC_HS_COMPLETE; ldc_set_state(lp, LDC_STATE_CONNECTED); - event_mask |= LDC_EVENT_UP; + /* + * Generate an LDC_EVENT_UP event if the channel + * was not already up. + */ + if (orig_state != LDC_CHANNEL_UP) { + event_mask |= LDC_EVENT_UP; + orig_state = lp->chan_state; + } } spin_unlock_irqrestore(&lp->lock, flags); @@ -998,31 +1009,59 @@ static void free_queue(unsigned long num_entries, struct ldc_packet *q) free_pages((unsigned long)q, order); } +static unsigned long ldc_cookie_to_index(u64 cookie, void *arg) +{ + u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT; + /* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */ + + cookie &= ~COOKIE_PGSZ_CODE; + + return (cookie >> (13ULL + (szcode * 3ULL))); +} + +static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie, + unsigned long entry, unsigned long npages) +{ + struct ldc_mtable_entry *base; + unsigned long i, shift; + + shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3; + base = iommu->page_table + entry; + for (i = 0; i < npages; i++) { + if (base->cookie) + sun4v_ldc_revoke(id, cookie + (i << shift), + base->cookie); + base->mte = 0; + } +} + /* XXX Make this configurable... XXX */ #define LDC_IOTABLE_SIZE (8 * 1024) -static int ldc_iommu_init(struct ldc_channel *lp) +static int ldc_iommu_init(const char *name, struct ldc_channel *lp) { unsigned long sz, num_tsb_entries, tsbsize, order; - struct ldc_iommu *iommu = &lp->iommu; + struct ldc_iommu *ldc_iommu = &lp->iommu; + struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table; struct ldc_mtable_entry *table; unsigned long hv_err; int err; num_tsb_entries = LDC_IOTABLE_SIZE; tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); - - spin_lock_init(&iommu->lock); + spin_lock_init(&ldc_iommu->lock); sz = num_tsb_entries / 8; sz = (sz + 7UL) & ~7UL; - iommu->arena.map = kzalloc(sz, GFP_KERNEL); - if (!iommu->arena.map) { + iommu->map = kzalloc(sz, GFP_KERNEL); + if (!iommu->map) { printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz); return -ENOMEM; } - - iommu->arena.limit = num_tsb_entries; + iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT, + NULL, false /* no large pool */, + 1 /* npools */, + true /* skip span boundary check */); order = get_order(tsbsize); @@ -1037,7 +1076,7 @@ static int ldc_iommu_init(struct ldc_channel *lp) memset(table, 0, PAGE_SIZE << order); - iommu->page_table = table; + ldc_iommu->page_table = table; hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table), num_tsb_entries); @@ -1049,36 +1088,38 @@ static int ldc_iommu_init(struct ldc_channel *lp) out_free_table: free_pages((unsigned long) table, order); - iommu->page_table = NULL; + ldc_iommu->page_table = NULL; out_free_map: - kfree(iommu->arena.map); - iommu->arena.map = NULL; + kfree(iommu->map); + iommu->map = NULL; return err; } static void ldc_iommu_release(struct ldc_channel *lp) { - struct ldc_iommu *iommu = &lp->iommu; + struct ldc_iommu *ldc_iommu = &lp->iommu; + struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table; unsigned long num_tsb_entries, tsbsize, order; (void) sun4v_ldc_set_map_table(lp->id, 0, 0); - num_tsb_entries = iommu->arena.limit; + num_tsb_entries = iommu->poolsize * iommu->nr_pools; tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); order = get_order(tsbsize); - free_pages((unsigned long) iommu->page_table, order); - iommu->page_table = NULL; + free_pages((unsigned long) ldc_iommu->page_table, order); + ldc_iommu->page_table = NULL; - kfree(iommu->arena.map); - iommu->arena.map = NULL; + kfree(iommu->map); + iommu->map = NULL; } struct ldc_channel *ldc_alloc(unsigned long id, const struct ldc_channel_config *cfgp, - void *event_arg) + void *event_arg, + const char *name) { struct ldc_channel *lp; const struct ldc_mode_ops *mops; @@ -1093,6 +1134,8 @@ struct ldc_channel *ldc_alloc(unsigned long id, err = -EINVAL; if (!cfgp) goto out_err; + if (!name) + goto out_err; switch (cfgp->mode) { case LDC_MODE_RAW: @@ -1137,7 +1180,7 @@ struct ldc_channel *ldc_alloc(unsigned long id, lp->id = id; - err = ldc_iommu_init(lp); + err = ldc_iommu_init(name, lp); if (err) goto out_free_ldc; @@ -1185,6 +1228,21 @@ struct ldc_channel *ldc_alloc(unsigned long id, INIT_HLIST_HEAD(&lp->mh_list); + snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name); + snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name); + + err = request_irq(lp->cfg.rx_irq, ldc_rx, 0, + lp->rx_irq_name, lp); + if (err) + goto out_free_txq; + + err = request_irq(lp->cfg.tx_irq, ldc_tx, 0, + lp->tx_irq_name, lp); + if (err) { + free_irq(lp->cfg.rx_irq, lp); + goto out_free_txq; + } + return lp; out_free_txq: @@ -1204,11 +1262,12 @@ out_err: } EXPORT_SYMBOL(ldc_alloc); -void ldc_free(struct ldc_channel *lp) +void ldc_unbind(struct ldc_channel *lp) { if (lp->flags & LDC_FLAG_REGISTERED_IRQS) { free_irq(lp->cfg.rx_irq, lp); free_irq(lp->cfg.tx_irq, lp); + lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; } if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) { @@ -1222,10 +1281,15 @@ void ldc_free(struct ldc_channel *lp) lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES; } - hlist_del(&lp->list); + ldc_set_state(lp, LDC_STATE_INIT); +} +EXPORT_SYMBOL(ldc_unbind); +void ldc_free(struct ldc_channel *lp) +{ + ldc_unbind(lp); + hlist_del(&lp->list); kfree(lp->mssbuf); - ldc_iommu_release(lp); kfree(lp); @@ -1237,31 +1301,14 @@ EXPORT_SYMBOL(ldc_free); * state. This does not initiate a handshake, ldc_connect() does * that. */ -int ldc_bind(struct ldc_channel *lp, const char *name) +int ldc_bind(struct ldc_channel *lp) { unsigned long hv_err, flags; int err = -EINVAL; - if (!name || - (lp->state != LDC_STATE_INIT)) + if (lp->state != LDC_STATE_INIT) return -EINVAL; - snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name); - snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name); - - err = request_irq(lp->cfg.rx_irq, ldc_rx, IRQF_DISABLED, - lp->rx_irq_name, lp); - if (err) - return err; - - err = request_irq(lp->cfg.tx_irq, ldc_tx, IRQF_DISABLED, - lp->tx_irq_name, lp); - if (err) { - free_irq(lp->cfg.rx_irq, lp); - return err; - } - - spin_lock_irqsave(&lp->lock, flags); enable_irq(lp->cfg.rx_irq); @@ -1301,6 +1348,14 @@ int ldc_bind(struct ldc_channel *lp, const char *name) lp->hs_state = LDC_HS_OPEN; ldc_set_state(lp, LDC_STATE_BOUND); + if (lp->cfg.mode == LDC_MODE_RAW) { + /* + * There is no handshake in RAW mode, so handshake + * is completed. + */ + lp->hs_state = LDC_HS_COMPLETE; + } + spin_unlock_irqrestore(&lp->lock, flags); return 0; @@ -1336,7 +1391,7 @@ int ldc_connect(struct ldc_channel *lp) if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) || lp->hs_state != LDC_HS_OPEN) - err = -EINVAL; + err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL); else err = start_handshake(lp); @@ -1406,12 +1461,56 @@ int ldc_state(struct ldc_channel *lp) } EXPORT_SYMBOL(ldc_state); +void ldc_set_state(struct ldc_channel *lp, u8 state) +{ + ldcdbg(STATE, "STATE (%s) --> (%s)\n", + state_to_str(lp->state), + state_to_str(state)); + + lp->state = state; +} +EXPORT_SYMBOL(ldc_set_state); + +int ldc_mode(struct ldc_channel *lp) +{ + return lp->cfg.mode; +} +EXPORT_SYMBOL(ldc_mode); + +int ldc_rx_reset(struct ldc_channel *lp) +{ + return __set_rx_head(lp, lp->rx_tail); +} +EXPORT_SYMBOL(ldc_rx_reset); + +void __ldc_print(struct ldc_channel *lp, const char *caller) +{ + pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n" + "\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n" + "\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n" + "\trcv_nxt=%u snd_nxt=%u\n", + caller, lp->id, lp->flags, state_to_str(lp->state), + lp->chan_state, lp->hs_state, + lp->rx_head, lp->rx_tail, lp->rx_num_entries, + lp->tx_head, lp->tx_tail, lp->tx_num_entries, + lp->rcv_nxt, lp->snd_nxt); +} +EXPORT_SYMBOL(__ldc_print); + static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size) { struct ldc_packet *p; - unsigned long new_tail; + unsigned long new_tail, hv_err; int err; + hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, + &lp->chan_state); + if (unlikely(hv_err)) + return -EBUSY; + + if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) + return LDC_ABORT(lp); + if (size > LDC_PACKET_SIZE) return -EMSGSIZE; @@ -1442,7 +1541,7 @@ static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size) &lp->rx_tail, &lp->chan_state); if (hv_err) - return ldc_abort(lp); + return LDC_ABORT(lp); if (lp->chan_state == LDC_CHANNEL_DOWN || lp->chan_state == LDC_CHANNEL_RESETTING) @@ -1485,7 +1584,7 @@ static int write_nonraw(struct ldc_channel *lp, const void *buf, return -EBUSY; if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) - return ldc_abort(lp); + return LDC_ABORT(lp); if (!tx_has_space_for(lp, size)) return -EAGAIN; @@ -1551,9 +1650,9 @@ static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p, if (err) return err; - err = __set_rx_head(lp, lp->rx_tail); + err = ldc_rx_reset(lp); if (err < 0) - return ldc_abort(lp); + return LDC_ABORT(lp); return 0; } @@ -1566,7 +1665,7 @@ static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p) return err; } if (p->stype & LDC_NACK) - return ldc_abort(lp); + return LDC_ABORT(lp); return 0; } @@ -1586,7 +1685,7 @@ static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head) &lp->rx_tail, &lp->chan_state); if (hv_err) - return ldc_abort(lp); + return LDC_ABORT(lp); if (lp->chan_state == LDC_CHANNEL_DOWN || lp->chan_state == LDC_CHANNEL_RESETTING) @@ -1609,7 +1708,7 @@ static int rx_set_head(struct ldc_channel *lp, unsigned long head) int err = __set_rx_head(lp, head); if (err < 0) - return ldc_abort(lp); + return LDC_ABORT(lp); lp->rx_head = head; return 0; @@ -1648,7 +1747,7 @@ static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size) &lp->rx_tail, &lp->chan_state); if (hv_err) - return ldc_abort(lp); + return LDC_ABORT(lp); if (lp->chan_state == LDC_CHANNEL_DOWN || lp->chan_state == LDC_CHANNEL_RESETTING) @@ -1692,9 +1791,14 @@ static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size) lp->rcv_nxt = p->seqid; + /* + * If this is a control-only packet, there is nothing + * else to do but advance the rx queue since the packet + * was already processed above. + */ if (!(p->type & LDC_DATA)) { new = rx_advance(lp, new); - goto no_data; + break; } if (p->stype & (LDC_ACK | LDC_NACK)) { err = data_ack_nack(lp, p); @@ -1750,7 +1854,7 @@ static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size) * This seems the best behavior because this allows * a user of the LDC layer to start with a small * RX buffer for ldc_read() calls and use -EMSGSIZE - * as a cue to enlarge it's read buffer. + * as a cue to enlarge its read buffer. */ err = -EMSGSIZE; break; @@ -1859,6 +1963,8 @@ int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size) unsigned long flags; int err; + ldcdbg(RX, "%s: entered size=%d\n", __func__, size); + if (!buf) return -EINVAL; @@ -1874,44 +1980,13 @@ int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size) spin_unlock_irqrestore(&lp->lock, flags); + ldcdbg(RX, "%s: mode=%d, head=%lu, tail=%lu rv=%d\n", __func__, + lp->cfg.mode, lp->rx_head, lp->rx_tail, err); + return err; } EXPORT_SYMBOL(ldc_read); -static long arena_alloc(struct ldc_iommu *iommu, unsigned long npages) -{ - struct iommu_arena *arena = &iommu->arena; - unsigned long n, start, end, limit; - int pass; - - limit = arena->limit; - start = arena->hint; - pass = 0; - -again: - n = bitmap_find_next_zero_area(arena->map, limit, start, npages, 0); - end = n + npages; - if (unlikely(end >= limit)) { - if (likely(pass < 1)) { - limit = start; - start = 0; - pass++; - goto again; - } else { - /* Scanned the whole thing, give up. */ - return -1; - } - } - bitmap_set(arena->map, n, npages); - - arena->hint = end; - - return n; -} - -#define COOKIE_PGSZ_CODE 0xf000000000000000ULL -#define COOKIE_PGSZ_CODE_SHIFT 60ULL - static u64 pagesize_code(void) { switch (PAGE_SIZE) { @@ -1938,24 +2013,15 @@ static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset) page_offset); } -static u64 cookie_to_index(u64 cookie, unsigned long *shift) -{ - u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT; - - cookie &= ~COOKIE_PGSZ_CODE; - - *shift = szcode * 3; - - return (cookie >> (13ULL + (szcode * 3ULL))); -} static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu, unsigned long npages) { long entry; - entry = arena_alloc(iommu, npages); - if (unlikely(entry < 0)) + entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table, + npages, NULL, (unsigned long)-1, 0); + if (unlikely(entry == IOMMU_ERROR_CODE)) return NULL; return iommu->page_table + entry; @@ -2083,11 +2149,12 @@ int ldc_map_sg(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, int ncookies, unsigned int map_perm) { - unsigned long i, npages, flags; + unsigned long i, npages; struct ldc_mtable_entry *base; struct cookie_state state; struct ldc_iommu *iommu; int err; + struct scatterlist *s; if (map_perm & ~LDC_MAP_ALL) return -EINVAL; @@ -2102,9 +2169,7 @@ int ldc_map_sg(struct ldc_channel *lp, iommu = &lp->iommu; - spin_lock_irqsave(&iommu->lock, flags); base = alloc_npages(iommu, npages); - spin_unlock_irqrestore(&iommu->lock, flags); if (!base) return -ENOMEM; @@ -2116,9 +2181,10 @@ int ldc_map_sg(struct ldc_channel *lp, state.pte_idx = (base - iommu->page_table); state.nc = 0; - for (i = 0; i < num_sg; i++) - fill_cookies(&state, page_to_pfn(sg_page(&sg[i])) << PAGE_SHIFT, - sg[i].offset, sg[i].length); + for_each_sg(sg, s, num_sg, i) { + fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT, + s->offset, s->length); + } return state.nc; } @@ -2129,7 +2195,7 @@ int ldc_map_single(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, int ncookies, unsigned int map_perm) { - unsigned long npages, pa, flags; + unsigned long npages, pa; struct ldc_mtable_entry *base; struct cookie_state state; struct ldc_iommu *iommu; @@ -2145,9 +2211,7 @@ int ldc_map_single(struct ldc_channel *lp, iommu = &lp->iommu; - spin_lock_irqsave(&iommu->lock, flags); base = alloc_npages(iommu, npages); - spin_unlock_irqrestore(&iommu->lock, flags); if (!base) return -ENOMEM; @@ -2159,41 +2223,31 @@ int ldc_map_single(struct ldc_channel *lp, state.pte_idx = (base - iommu->page_table); state.nc = 0; fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len); - BUG_ON(state.nc != 1); + BUG_ON(state.nc > ncookies); return state.nc; } EXPORT_SYMBOL(ldc_map_single); + static void free_npages(unsigned long id, struct ldc_iommu *iommu, u64 cookie, u64 size) { - struct iommu_arena *arena = &iommu->arena; - unsigned long i, shift, index, npages; - struct ldc_mtable_entry *base; + unsigned long npages, entry; npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT; - index = cookie_to_index(cookie, &shift); - base = iommu->page_table + index; - BUG_ON(index > arena->limit || - (index + npages) > arena->limit); - - for (i = 0; i < npages; i++) { - if (base->cookie) - sun4v_ldc_revoke(id, cookie + (i << shift), - base->cookie); - base->mte = 0; - __clear_bit(index + i, arena->map); - } + entry = ldc_cookie_to_index(cookie, iommu); + ldc_demap(iommu, id, cookie, entry, npages); + iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry); } void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, int ncookies) { struct ldc_iommu *iommu = &lp->iommu; - unsigned long flags; int i; + unsigned long flags; spin_lock_irqsave(&iommu->lock, flags); for (i = 0; i < ncookies; i++) { @@ -2306,7 +2360,7 @@ void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len, if (len & (8UL - 1)) return ERR_PTR(-EINVAL); - buf = kzalloc(len, GFP_KERNEL); + buf = kzalloc(len, GFP_ATOMIC); if (!buf) return ERR_PTR(-ENOMEM); diff --git a/arch/sparc/kernel/led.c b/arch/sparc/kernel/led.c index 3ae36f36e758..f4fb82b019bb 100644 --- a/arch/sparc/kernel/led.c +++ b/arch/sparc/kernel/led.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> @@ -8,6 +9,7 @@ #include <linux/jiffies.h> #include <linux/timer.h> #include <linux/uaccess.h> +#include <linux/sched/loadavg.h> #include <asm/auxio.h> @@ -30,23 +32,25 @@ static inline void led_toggle(void) } static struct timer_list led_blink_timer; +static unsigned long led_blink_timer_timeout; -static void led_blink(unsigned long timeout) +static void led_blink(struct timer_list *unused) { + unsigned long timeout = led_blink_timer_timeout; + led_toggle(); /* reschedule */ if (!timeout) { /* blink according to load */ led_blink_timer.expires = jiffies + ((1 + (avenrun[0] >> FSHIFT)) * HZ); - led_blink_timer.data = 0; } else { /* blink at user specified interval */ led_blink_timer.expires = jiffies + (timeout * HZ); - led_blink_timer.data = timeout; } add_timer(&led_blink_timer); } +#ifdef CONFIG_PROC_FS static int led_proc_show(struct seq_file *m, void *v) { if (get_auxio() & AUXIO_LED) @@ -69,16 +73,9 @@ static ssize_t led_proc_write(struct file *file, const char __user *buffer, if (count > LED_MAX_LENGTH) count = LED_MAX_LENGTH; - buf = kmalloc(sizeof(char) * (count + 1), GFP_KERNEL); - if (!buf) - return -ENOMEM; - - if (copy_from_user(buf, buffer, count)) { - kfree(buf); - return -EFAULT; - } - - buf[count] = '\0'; + buf = memdup_user_nul(buffer, count); + if (IS_ERR(buf)) + return PTR_ERR(buf); /* work around \n when echo'ing into proc */ if (buf[count - 1] == '\n') @@ -87,16 +84,18 @@ static ssize_t led_proc_write(struct file *file, const char __user *buffer, /* before we change anything we want to stop any running timers, * otherwise calls such as on will have no persistent effect */ - del_timer_sync(&led_blink_timer); + timer_delete_sync(&led_blink_timer); if (!strcmp(buf, "on")) { auxio_set_led(AUXIO_LED_ON); } else if (!strcmp(buf, "toggle")) { led_toggle(); } else if ((*buf > '0') && (*buf <= '9')) { - led_blink(simple_strtoul(buf, NULL, 10)); + led_blink_timer_timeout = simple_strtoul(buf, NULL, 10); + led_blink(&led_blink_timer); } else if (!strcmp(buf, "load")) { - led_blink(0); + led_blink_timer_timeout = 0; + led_blink(&led_blink_timer); } else { auxio_set_led(AUXIO_LED_OFF); } @@ -106,28 +105,25 @@ static ssize_t led_proc_write(struct file *file, const char __user *buffer, return count; } -static const struct file_operations led_proc_fops = { - .owner = THIS_MODULE, - .open = led_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = led_proc_write, +static const struct proc_ops led_proc_ops = { + .proc_open = led_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = led_proc_write, }; - -static struct proc_dir_entry *led; +#endif #define LED_VERSION "0.1" static int __init led_init(void) { - init_timer(&led_blink_timer); - led_blink_timer.function = led_blink; + timer_setup(&led_blink_timer, led_blink, 0); - led = proc_create("led", 0, NULL, &led_proc_fops); - if (!led) +#ifdef CONFIG_PROC_FS + if (!proc_create("led", 0, NULL, &led_proc_ops)) return -ENOMEM; - +#endif printk(KERN_INFO "led: version %s, Lars Kotthoff <metalhead@metalhead.ws>\n", LED_VERSION); @@ -138,7 +134,7 @@ static int __init led_init(void) static void __exit led_exit(void) { remove_proc_entry("led", NULL); - del_timer_sync(&led_blink_timer); + timer_delete_sync(&led_blink_timer); } module_init(led_init); diff --git a/arch/sparc/kernel/leon_kernel.c b/arch/sparc/kernel/leon_kernel.c index b7c68976cbc7..a43cf794bb1e 100644 --- a/arch/sparc/kernel/leon_kernel.c +++ b/arch/sparc/kernel/leon_kernel.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2009 Daniel Hellstrom (daniel@gaisler.com) Aeroflex Gaisler AB * Copyright (C) 2009 Konrad Eisele (konrad@gaisler.com) Aeroflex Gaisler AB @@ -7,9 +8,7 @@ #include <linux/errno.h> #include <linux/mutex.h> #include <linux/of.h> -#include <linux/of_platform.h> #include <linux/interrupt.h> -#include <linux/of_device.h> #include <linux/clocksource.h> #include <linux/clockchips.h> @@ -32,12 +31,13 @@ struct leon3_gptimer_regs_map *leon3_gptimer_regs; /* timer controller base addr int leondebug_irq_disable; int leon_debug_irqout; -static int dummy_master_l10_counter; +static volatile u32 dummy_master_l10_counter; unsigned long amba_system_id; static DEFINE_SPINLOCK(leon_irq_lock); +static unsigned long leon3_gptimer_idx; /* Timer Index (0..6) within Timer Core */ +static unsigned long leon3_gptimer_ackmask; /* For clearing pending bit */ unsigned long leon3_gptimer_irq; /* interrupt controller irq number */ -unsigned long leon3_gptimer_idx; /* Timer Index (0..6) within Timer Core */ unsigned int sparc_leon_eirq; #define LEON_IMASK(cpu) (&leon3_irqctrl_regs->mask[cpu]) #define LEON_IACK (&leon3_irqctrl_regs->iclear) @@ -52,7 +52,7 @@ static inline unsigned int leon_eirq_get(int cpu) } /* Handle one or multiple IRQs from the extended interrupt controller */ -static void leon_handle_ext_irq(unsigned int irq, struct irq_desc *desc) +static void leon_handle_ext_irq(struct irq_desc *desc) { unsigned int eirq; struct irq_bucket *p; @@ -65,7 +65,7 @@ static void leon_handle_ext_irq(unsigned int irq, struct irq_desc *desc) } /* The extended IRQ controller has been found, this function registers it */ -void leon_eirq_setup(unsigned int eirq) +static void leon_eirq_setup(unsigned int eirq) { unsigned long mask, oldmask; unsigned int veirq; @@ -106,13 +106,12 @@ unsigned long leon_get_irqmask(unsigned int irq) #ifdef CONFIG_SMP static int irq_choose_cpu(const struct cpumask *affinity) { - cpumask_t mask; + unsigned int cpu = cpumask_first_and(affinity, cpu_online_mask); - cpumask_and(&mask, cpu_online_mask, affinity); - if (cpumask_equal(&mask, cpu_online_mask) || cpumask_empty(&mask)) + if (cpumask_subset(cpu_online_mask, affinity) || cpu >= nr_cpu_ids) return boot_cpu_id; else - return cpumask_first(&mask); + return cpu; } #else #define irq_choose_cpu(affinity) boot_cpu_id @@ -125,7 +124,7 @@ static int leon_set_affinity(struct irq_data *data, const struct cpumask *dest, int oldcpu, newcpu; mask = (unsigned long)data->chip_data; - oldcpu = irq_choose_cpu(data->affinity); + oldcpu = irq_choose_cpu(irq_data_get_affinity_mask(data)); newcpu = irq_choose_cpu(dest); if (oldcpu == newcpu) @@ -148,7 +147,7 @@ static void leon_unmask_irq(struct irq_data *data) int cpu; mask = (unsigned long)data->chip_data; - cpu = irq_choose_cpu(data->affinity); + cpu = irq_choose_cpu(irq_data_get_affinity_mask(data)); spin_lock_irqsave(&leon_irq_lock, flags); oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(cpu)); LEON3_BYPASS_STORE_PA(LEON_IMASK(cpu), (oldmask | mask)); @@ -161,7 +160,7 @@ static void leon_mask_irq(struct irq_data *data) int cpu; mask = (unsigned long)data->chip_data; - cpu = irq_choose_cpu(data->affinity); + cpu = irq_choose_cpu(irq_data_get_affinity_mask(data)); spin_lock_irqsave(&leon_irq_lock, flags); oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(cpu)); LEON3_BYPASS_STORE_PA(LEON_IMASK(cpu), (oldmask & ~mask)); @@ -202,7 +201,7 @@ static struct irq_chip leon_irq = { /* * Build a LEON IRQ for the edge triggered LEON IRQ controller: - * Edge (normal) IRQ - handle_simple_irq, ack=DONT-CARE, never ack + * Edge (normal) IRQ - handle_simple_irq, ack=DON'T-CARE, never ack * Level IRQ (PCI|Level-GPIO) - handle_fasteoi_irq, ack=1, ack after ISR * Per-CPU Edge - handle_percpu_irq, ack=0 */ @@ -260,17 +259,25 @@ void leon_update_virq_handling(unsigned int virq, static u32 leon_cycles_offset(void) { - u32 rld, val, off; + u32 rld, val, ctrl, off; + rld = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld); val = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val); - off = rld - val; - return rld - val; + ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl); + if (LEON3_GPTIMER_CTRL_ISPENDING(ctrl)) { + val = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val); + off = 2 * rld - val; + } else { + off = rld - val; + } + + return off; } #ifdef CONFIG_SMP /* smp clockevent irq */ -irqreturn_t leon_percpu_timer_ce_interrupt(int irq, void *unused) +static irqreturn_t leon_percpu_timer_ce_interrupt(int irq, void *unused) { struct clock_event_device *ce; int cpu = smp_processor_id(); @@ -302,6 +309,7 @@ void __init leon_init_timers(void) int ampopts; int err; u32 config; + u32 ctrl; sparc_config.get_cycles_offset = leon_cycles_offset; sparc_config.cs_period = 1000000 / HZ; @@ -313,7 +321,7 @@ void __init leon_init_timers(void) leondebug_irq_disable = 0; leon_debug_irqout = 0; - master_l10_counter = (unsigned int *)&dummy_master_l10_counter; + master_l10_counter = (u32 __iomem *)&dummy_master_l10_counter; dummy_master_l10_counter = 0; rootnp = of_find_node_by_path("/ambapp0"); @@ -339,41 +347,51 @@ void __init leon_init_timers(void) /* Find GPTIMER Timer Registers base address otherwise bail out. */ nnp = rootnp; - do { - np = of_find_node_by_name(nnp, "GAISLER_GPTIMER"); - if (!np) { - np = of_find_node_by_name(nnp, "01_011"); - if (!np) - goto bad; - } - ampopts = 0; - pp = of_find_property(np, "ampopts", &len); - if (pp) { - ampopts = *(int *)pp->value; - if (ampopts == 0) { - /* Skip this instance, resource already - * allocated by other OS */ - nnp = np; - continue; - } +retry: + np = of_find_node_by_name(nnp, "GAISLER_GPTIMER"); + if (!np) { + np = of_find_node_by_name(nnp, "01_011"); + if (!np) + goto bad; + } + + ampopts = 0; + pp = of_find_property(np, "ampopts", &len); + if (pp) { + ampopts = *(int *)pp->value; + if (ampopts == 0) { + /* Skip this instance, resource already + * allocated by other OS */ + nnp = np; + goto retry; } + } - /* Select Timer-Instance on Timer Core. Default is zero */ - leon3_gptimer_idx = ampopts & 0x7; + /* Select Timer-Instance on Timer Core. Default is zero */ + leon3_gptimer_idx = ampopts & 0x7; - pp = of_find_property(np, "reg", &len); - if (pp) - leon3_gptimer_regs = *(struct leon3_gptimer_regs_map **) - pp->value; - pp = of_find_property(np, "interrupts", &len); - if (pp) - leon3_gptimer_irq = *(unsigned int *)pp->value; - } while (0); + pp = of_find_property(np, "reg", &len); + if (pp) + leon3_gptimer_regs = *(struct leon3_gptimer_regs_map **) + pp->value; + pp = of_find_property(np, "interrupts", &len); + if (pp) + leon3_gptimer_irq = *(unsigned int *)pp->value; if (!(leon3_gptimer_regs && leon3_irqctrl_regs && leon3_gptimer_irq)) goto bad; + ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl); + LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl, + ctrl | LEON3_GPTIMER_CTRL_PENDING); + ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl); + + if ((ctrl & LEON3_GPTIMER_CTRL_PENDING) != 0) + leon3_gptimer_ackmask = ~LEON3_GPTIMER_CTRL_PENDING; + else + leon3_gptimer_ackmask = ~0; + LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val, 0); LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld, (((1000000 / HZ) - 1))); @@ -452,24 +470,15 @@ bad: static void leon_clear_clock_irq(void) { -} + u32 ctrl; -static void leon_load_profile_irq(int cpu, unsigned int limit) -{ + ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl); + LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl, + ctrl & leon3_gptimer_ackmask); } -void __init leon_trans_init(struct device_node *dp) +static void leon_load_profile_irq(int cpu, unsigned int limit) { - if (strcmp(dp->type, "cpu") == 0 && strcmp(dp->name, "<NULL>") == 0) { - struct property *p; - p = of_find_property(dp, "mid", (void *)0); - if (p) { - int mid; - dp->name = prom_early_alloc(5 + 1); - memcpy(&mid, p->value, p->length); - sprintf((char *)dp->name, "cpu%.2d", mid); - } - } } #ifdef CONFIG_SMP diff --git a/arch/sparc/kernel/leon_pci.c b/arch/sparc/kernel/leon_pci.c index 88aaaa57bb64..10934dfa987a 100644 --- a/arch/sparc/kernel/leon_pci.c +++ b/arch/sparc/kernel/leon_pci.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * leon_pci.c: LEON Host PCI support * @@ -6,7 +7,8 @@ * Code is partially derived from pcic.c */ -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/platform_device.h> #include <linux/kernel.h> #include <linux/pci.h> #include <linux/export.h> @@ -25,6 +27,12 @@ void leon_pci_init(struct platform_device *ofdev, struct leon_pci_info *info) { LIST_HEAD(resources); struct pci_bus *root_bus; + struct pci_host_bridge *bridge; + int ret; + + bridge = pci_alloc_host_bridge(0); + if (!bridge) + return; pci_add_resource_offset(&resources, &info->io_space, info->io_space.start - 0x1000); @@ -32,153 +40,23 @@ void leon_pci_init(struct platform_device *ofdev, struct leon_pci_info *info) info->busn.flags = IORESOURCE_BUS; pci_add_resource(&resources, &info->busn); - root_bus = pci_scan_root_bus(&ofdev->dev, 0, info->ops, info, - &resources); - if (root_bus) { - /* Setup IRQs of all devices using custom routines */ - pci_fixup_irqs(pci_common_swizzle, info->map_irq); - - /* Assign devices with resources */ - pci_assign_unassigned_resources(); - } else { - pci_free_resource_list(&resources); - } -} - -void pcibios_fixup_bus(struct pci_bus *pbus) -{ - struct pci_dev *dev; - int i, has_io, has_mem; - u16 cmd; + list_splice_init(&resources, &bridge->windows); + bridge->dev.parent = &ofdev->dev; + bridge->sysdata = info; + bridge->busnr = 0; + bridge->ops = info->ops; + bridge->swizzle_irq = pci_common_swizzle; + bridge->map_irq = info->map_irq; - list_for_each_entry(dev, &pbus->devices, bus_list) { - /* - * We can not rely on that the bootloader has enabled I/O - * or memory access to PCI devices. Instead we enable it here - * if the device has BARs of respective type. - */ - has_io = has_mem = 0; - for (i = 0; i < PCI_ROM_RESOURCE; i++) { - unsigned long f = dev->resource[i].flags; - if (f & IORESOURCE_IO) - has_io = 1; - else if (f & IORESOURCE_MEM) - has_mem = 1; - } - /* ROM BARs are mapped into 32-bit memory space */ - if (dev->resource[PCI_ROM_RESOURCE].end != 0) { - dev->resource[PCI_ROM_RESOURCE].flags |= - IORESOURCE_ROM_ENABLE; - has_mem = 1; - } - pci_bus_read_config_word(pbus, dev->devfn, PCI_COMMAND, &cmd); - if (has_io && !(cmd & PCI_COMMAND_IO)) { -#ifdef CONFIG_PCI_DEBUG - printk(KERN_INFO "LEONPCI: Enabling I/O for dev %s\n", - pci_name(dev)); -#endif - cmd |= PCI_COMMAND_IO; - pci_bus_write_config_word(pbus, dev->devfn, PCI_COMMAND, - cmd); - } - if (has_mem && !(cmd & PCI_COMMAND_MEMORY)) { -#ifdef CONFIG_PCI_DEBUG - printk(KERN_INFO "LEONPCI: Enabling MEMORY for dev" - "%s\n", pci_name(dev)); -#endif - cmd |= PCI_COMMAND_MEMORY; - pci_bus_write_config_word(pbus, dev->devfn, PCI_COMMAND, - cmd); - } + ret = pci_scan_root_bus_bridge(bridge); + if (ret) { + pci_free_host_bridge(bridge); + return; } -} - -resource_size_t pcibios_align_resource(void *data, const struct resource *res, - resource_size_t size, resource_size_t align) -{ - return res->start; -} -int pcibios_enable_device(struct pci_dev *dev, int mask) -{ - return pci_enable_resources(dev, mask); -} - -/* in/out routines taken from pcic.c - * - * This probably belongs here rather than ioport.c because - * we do not want this crud linked into SBus kernels. - * Also, think for a moment about likes of floppy.c that - * include architecture specific parts. They may want to redefine ins/outs. - * - * We do not use horrible macros here because we want to - * advance pointer by sizeof(size). - */ -void outsb(unsigned long addr, const void *src, unsigned long count) -{ - while (count) { - count -= 1; - outb(*(const char *)src, addr); - src += 1; - /* addr += 1; */ - } -} -EXPORT_SYMBOL(outsb); + root_bus = bridge->bus; -void outsw(unsigned long addr, const void *src, unsigned long count) -{ - while (count) { - count -= 2; - outw(*(const short *)src, addr); - src += 2; - /* addr += 2; */ - } -} -EXPORT_SYMBOL(outsw); - -void outsl(unsigned long addr, const void *src, unsigned long count) -{ - while (count) { - count -= 4; - outl(*(const long *)src, addr); - src += 4; - /* addr += 4; */ - } -} -EXPORT_SYMBOL(outsl); - -void insb(unsigned long addr, void *dst, unsigned long count) -{ - while (count) { - count -= 1; - *(unsigned char *)dst = inb(addr); - dst += 1; - /* addr += 1; */ - } -} -EXPORT_SYMBOL(insb); - -void insw(unsigned long addr, void *dst, unsigned long count) -{ - while (count) { - count -= 2; - *(unsigned short *)dst = inw(addr); - dst += 2; - /* addr += 2; */ - } -} -EXPORT_SYMBOL(insw); - -void insl(unsigned long addr, void *dst, unsigned long count) -{ - while (count) { - count -= 4; - /* - * XXX I am sure we are in for an unaligned trap here. - */ - *(unsigned long *)dst = inl(addr); - dst += 4; - /* addr += 4; */ - } + /* Assign devices with resources */ + pci_assign_unassigned_resources(); + pci_bus_add_devices(root_bus); } -EXPORT_SYMBOL(insl); diff --git a/arch/sparc/kernel/leon_pci_grpci1.c b/arch/sparc/kernel/leon_pci_grpci1.c index 6df26e37f879..b2b639bee068 100644 --- a/arch/sparc/kernel/leon_pci_grpci1.c +++ b/arch/sparc/kernel/leon_pci_grpci1.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * leon_pci_grpci1.c: GRPCI1 Host PCI driver * @@ -12,10 +13,11 @@ * Contributors: Daniel Hellstrom <daniel@gaisler.com> */ -#include <linux/of_device.h> #include <linux/export.h> #include <linux/kernel.h> +#include <linux/of.h> #include <linux/of_irq.h> +#include <linux/platform_device.h> #include <linux/delay.h> #include <linux/pci.h> @@ -80,7 +82,7 @@ struct grpci1_regs { struct grpci1_priv { struct leon_pci_info info; /* must be on top of this structure */ - struct grpci1_regs *regs; /* GRPCI register map */ + struct grpci1_regs __iomem *regs; /* GRPCI register map */ struct device *dev; int pci_err_mask; /* STATUS register error mask */ int irq; /* LEON irqctrl GRPCI IRQ */ @@ -101,7 +103,7 @@ static struct grpci1_priv *grpci1priv; static int grpci1_cfg_w32(struct grpci1_priv *priv, unsigned int bus, unsigned int devfn, int where, u32 val); -int grpci1_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +static int grpci1_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) { struct grpci1_priv *priv = dev->bus->sysdata; int irq_group; @@ -144,7 +146,7 @@ static int grpci1_cfg_r32(struct grpci1_priv *priv, unsigned int bus, grpci1_cfg_w32(priv, TGT, 0, PCI_COMMAND, tmp); } else { /* Bus always little endian (unaffected by byte-swapping) */ - *val = flip_dword(tmp); + *val = swab32(tmp); } return 0; @@ -197,7 +199,7 @@ static int grpci1_cfg_w32(struct grpci1_priv *priv, unsigned int bus, pci_conf = (unsigned int *) (priv->pci_conf | (devfn << 8) | (where & 0xfc)); - LEON3_BYPASS_STORE_PA(pci_conf, flip_dword(val)); + LEON3_BYPASS_STORE_PA(pci_conf, swab32(val)); return 0; } @@ -357,7 +359,7 @@ static struct irq_chip grpci1_irq = { }; /* Handle one or multiple IRQs from the PCI core */ -static void grpci1_pci_flow_irq(unsigned int irq, struct irq_desc *desc) +static void grpci1_pci_flow_irq(struct irq_desc *desc) { struct grpci1_priv *priv = grpci1priv; int i, ack = 0; @@ -417,10 +419,10 @@ out: * BAR1: peripheral DMA to host's memory (size at least 256MByte) * BAR2..BAR5: not implemented in hardware */ -void grpci1_hw_init(struct grpci1_priv *priv) +static void grpci1_hw_init(struct grpci1_priv *priv) { u32 ahbadr, bar_sz, data, pciadr; - struct grpci1_regs *regs = priv->regs; + struct grpci1_regs __iomem *regs = priv->regs; /* set 1:1 mapping between AHB -> PCI memory space */ REGSTORE(regs->cfg_stat, priv->pci_area & 0xf0000000); @@ -509,7 +511,7 @@ static irqreturn_t grpci1_err_interrupt(int irq, void *arg) static int grpci1_of_probe(struct platform_device *ofdev) { - struct grpci1_regs *regs; + struct grpci1_regs __iomem *regs; struct grpci1_priv *priv; int err, len; const int *tmp; @@ -690,12 +692,12 @@ err3: err2: release_resource(&priv->info.mem_space); err1: - iounmap((void *)priv->pci_io_va); + iounmap((void __iomem *)priv->pci_io_va); grpci1priv = NULL; return err; } -static struct of_device_id grpci1_of_match[] = { +static const struct of_device_id grpci1_of_match[] = { { .name = "GAISLER_PCIFBRG", }, @@ -708,7 +710,6 @@ static struct of_device_id grpci1_of_match[] = { static struct platform_driver grpci1_of_driver = { .driver = { .name = "grpci1", - .owner = THIS_MODULE, .of_match_table = grpci1_of_match, }, .probe = grpci1_of_probe, diff --git a/arch/sparc/kernel/leon_pci_grpci2.c b/arch/sparc/kernel/leon_pci_grpci2.c index 5f0402aab7fb..9f662340b5b2 100644 --- a/arch/sparc/kernel/leon_pci_grpci2.c +++ b/arch/sparc/kernel/leon_pci_grpci2.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * leon_pci_grpci2.c: GRPCI2 Host PCI driver * @@ -5,11 +6,14 @@ * */ -#include <linux/of_device.h> #include <linux/kernel.h> #include <linux/pci.h> +#include <linux/slab.h> #include <linux/delay.h> #include <linux/export.h> +#include <linux/of.h> +#include <linux/platform_device.h> + #include <asm/io.h> #include <asm/leon.h> #include <asm/vaddrs.h> @@ -190,7 +194,7 @@ struct grpci2_cap_first { struct grpci2_priv { struct leon_pci_info info; /* must be on top of this structure */ - struct grpci2_regs *regs; + struct grpci2_regs __iomem *regs; char irq; char irq_mode; /* IRQ Mode from CAPSTS REG */ char bt_enabled; @@ -214,10 +218,10 @@ struct grpci2_priv { struct grpci2_barcfg tgtbars[6]; }; -DEFINE_SPINLOCK(grpci2_dev_lock); -struct grpci2_priv *grpci2priv; +static DEFINE_SPINLOCK(grpci2_dev_lock); +static struct grpci2_priv *grpci2priv; -int grpci2_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +static int grpci2_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) { struct grpci2_priv *priv = dev->bus->sysdata; int irq_group; @@ -269,7 +273,7 @@ static int grpci2_cfg_r32(struct grpci2_priv *priv, unsigned int bus, *val = 0xffffffff; } else { /* Bus always little endian (unaffected by byte-swapping) */ - *val = flip_dword(tmp); + *val = swab32(tmp); } return 0; @@ -327,7 +331,7 @@ static int grpci2_cfg_w32(struct grpci2_priv *priv, unsigned int bus, pci_conf = (unsigned int *) (priv->pci_conf | (devfn << 8) | (where & 0xfc)); - LEON3_BYPASS_STORE_PA(pci_conf, flip_dword(val)); + LEON3_BYPASS_STORE_PA(pci_conf, swab32(val)); /* Wait until GRPCI2 signals that CFG access is done, it should be * done instantaneously unless a DMA operation is ongoing... @@ -497,7 +501,7 @@ static struct irq_chip grpci2_irq = { }; /* Handle one or multiple IRQs from the PCI core */ -static void grpci2_pci_flow_irq(unsigned int irq, struct irq_desc *desc) +static void grpci2_pci_flow_irq(struct irq_desc *desc) { struct grpci2_priv *priv = grpci2priv; int i, ack = 0; @@ -560,10 +564,10 @@ out: return virq; } -void grpci2_hw_init(struct grpci2_priv *priv) +static void grpci2_hw_init(struct grpci2_priv *priv) { u32 ahbadr, pciadr, bar_sz, capptr, io_map, data; - struct grpci2_regs *regs = priv->regs; + struct grpci2_regs __iomem *regs = priv->regs; int i; struct grpci2_barcfg *barcfg = priv->tgtbars; @@ -582,7 +586,7 @@ void grpci2_hw_init(struct grpci2_priv *priv) REGSTORE(regs->io_map, REGLOAD(regs->io_map) & 0x0000ffff); /* set 1:1 mapping between AHB -> PCI memory space, for all Masters - * Each AHB master has it's own mapping registers. Max 16 AHB masters. + * Each AHB master has its own mapping registers. Max 16 AHB masters. */ for (i = 0; i < 16; i++) REGSTORE(regs->ahbmst_map[i], priv->pci_area); @@ -654,7 +658,7 @@ static irqreturn_t grpci2_jump_interrupt(int irq, void *arg) static irqreturn_t grpci2_err_interrupt(int irq, void *arg) { struct grpci2_priv *priv = arg; - struct grpci2_regs *regs = priv->regs; + struct grpci2_regs __iomem *regs = priv->regs; unsigned int status; status = REGLOAD(regs->sts_cap); @@ -681,7 +685,7 @@ static irqreturn_t grpci2_err_interrupt(int irq, void *arg) static int grpci2_of_probe(struct platform_device *ofdev) { - struct grpci2_regs *regs; + struct grpci2_regs __iomem *regs; struct grpci2_priv *priv; int err, i, len; const int *tmp; @@ -722,7 +726,6 @@ static int grpci2_of_probe(struct platform_device *ofdev) err = -ENOMEM; goto err1; } - memset(grpci2priv, 0, sizeof(*grpci2priv)); priv->regs = regs; priv->irq = ofdev->archdata.irqs[0]; /* BASE IRQ */ priv->irq_mode = (capability & STS_IRQMODE) >> STS_IRQMODE_BIT; @@ -877,7 +880,7 @@ err4: release_resource(&priv->info.mem_space); err3: err = -ENOMEM; - iounmap((void *)priv->pci_io_va); + iounmap((void __iomem *)priv->pci_io_va); err2: kfree(priv); err1: @@ -886,7 +889,7 @@ err1: return err; } -static struct of_device_id grpci2_of_match[] = { +static const struct of_device_id grpci2_of_match[] = { { .name = "GAISLER_GRPCI2", }, @@ -899,7 +902,6 @@ static struct of_device_id grpci2_of_match[] = { static struct platform_driver grpci2_of_driver = { .driver = { .name = "grpci2", - .owner = THIS_MODULE, .of_match_table = grpci2_of_match, }, .probe = grpci2_of_probe, diff --git a/arch/sparc/kernel/leon_pmc.c b/arch/sparc/kernel/leon_pmc.c index b0b3967a2dd2..6c00cbad7fb5 100644 --- a/arch/sparc/kernel/leon_pmc.c +++ b/arch/sparc/kernel/leon_pmc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* leon_pmc.c: LEON Power-down cpu_idle() handler * * Copyright (C) 2011 Daniel Hellstrom (daniel@gaisler.com) Aeroflex Gaisler AB @@ -12,14 +13,14 @@ #include <asm/processor.h> /* List of Systems that need fixup instructions around power-down instruction */ -unsigned int pmc_leon_fixup_ids[] = { +static unsigned int pmc_leon_fixup_ids[] = { AEROFLEX_UT699, GAISLER_GR712RC, LEON4_NEXTREME1, 0 }; -int pmc_leon_need_fixup(void) +static int pmc_leon_need_fixup(void) { unsigned int systemid = amba_system_id >> 16; unsigned int *id; @@ -38,7 +39,7 @@ int pmc_leon_need_fixup(void) * CPU idle callback function for systems that need some extra handling * See .../arch/sparc/kernel/process.c */ -void pmc_leon_idle_fixup(void) +static void pmc_leon_idle_fixup(void) { /* Prepare an address to a non-cachable region. APB is always * none-cachable. One instruction is executed after the Sleep @@ -49,26 +50,30 @@ void pmc_leon_idle_fixup(void) register unsigned int address = (unsigned int)leon3_irqctrl_regs; /* Interrupts need to be enabled to not hang the CPU */ - local_irq_enable(); + raw_local_irq_enable(); __asm__ __volatile__ ( "wr %%g0, %%asr19\n" "lda [%0] %1, %%g0\n" : : "r"(address), "i"(ASI_LEON_BYPASS)); + + raw_local_irq_disable(); } /* * CPU idle callback function * See .../arch/sparc/kernel/process.c */ -void pmc_leon_idle(void) +static void pmc_leon_idle(void) { /* Interrupts need to be enabled to not hang the CPU */ - local_irq_enable(); + raw_local_irq_enable(); /* For systems without power-down, this will be no-op */ __asm__ __volatile__ ("wr %g0, %asr19\n\t"); + + raw_local_irq_disable(); } /* Install LEON Power Down function */ diff --git a/arch/sparc/kernel/leon_smp.c b/arch/sparc/kernel/leon_smp.c index d7aa524b7283..1ee393abc463 100644 --- a/arch/sparc/kernel/leon_smp.c +++ b/arch/sparc/kernel/leon_smp.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* leon_smp.c: Sparc-Leon SMP support. * * based on sun4m_smp.c @@ -9,7 +10,7 @@ #include <asm/head.h> #include <linux/kernel.h> -#include <linux/sched.h> +#include <linux/sched/mm.h> #include <linux/threads.h> #include <linux/smp.h> #include <linux/interrupt.h> @@ -37,8 +38,6 @@ #include <asm/delay.h> #include <asm/irq.h> #include <asm/page.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> #include <asm/oplib.h> #include <asm/cpudata.h> #include <asm/asi.h> @@ -54,7 +53,7 @@ extern ctxd_t *srmmu_ctx_table_phys; static int smp_processors_ready; extern volatile unsigned long cpu_callin_map[NR_CPUS]; extern cpumask_t smp_commenced_mask; -void __cpuinit leon_configure_cache_smp(void); +void leon_configure_cache_smp(void); static void leon_ipi_init(void); /* IRQ number of LEON IPIs */ @@ -69,12 +68,12 @@ static inline unsigned long do_swap(volatile unsigned long *ptr, return val; } -void __cpuinit leon_cpu_pre_starting(void *arg) +void leon_cpu_pre_starting(void *arg) { leon_configure_cache_smp(); } -void __cpuinit leon_cpu_pre_online(void *arg) +void leon_cpu_pre_online(void *arg) { int cpuid = hard_smp_processor_id(); @@ -93,7 +92,7 @@ void __cpuinit leon_cpu_pre_online(void *arg) : "memory" /* paranoid */); /* Attach to the address space of init_task. */ - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; while (!cpumask_test_cpu(cpuid, &smp_commenced_mask)) @@ -106,7 +105,7 @@ void __cpuinit leon_cpu_pre_online(void *arg) extern struct linux_prom_registers smp_penguin_ctable; -void __cpuinit leon_configure_cache_smp(void) +void leon_configure_cache_smp(void) { unsigned long cfg = sparc_leon3_get_dcachecfg(); int me = smp_processor_id(); @@ -130,7 +129,7 @@ void __cpuinit leon_configure_cache_smp(void) local_ops->tlb_all(); } -void leon_smp_setbroadcast(unsigned int mask) +static void leon_smp_setbroadcast(unsigned int mask) { int broadcast = ((LEON3_BYPASS_LOAD_PA(&(leon3_irqctrl_regs->mpstatus)) >> @@ -148,13 +147,6 @@ void leon_smp_setbroadcast(unsigned int mask) LEON_BYPASS_STORE_PA(&(leon3_irqctrl_regs->mpbroadcast), mask); } -unsigned int leon_smp_getbroadcast(void) -{ - unsigned int mask; - mask = LEON_BYPASS_LOAD_PA(&(leon3_irqctrl_regs->mpbroadcast)); - return mask; -} - int leon_smp_nrcpus(void) { int nrcpu = @@ -186,7 +178,7 @@ void __init leon_boot_cpus(void) } -int __cpuinit leon_boot_one_cpu(int i, struct task_struct *idle) +int leon_boot_one_cpu(int i, struct task_struct *idle) { int timeout; @@ -253,23 +245,19 @@ void __init leon_smp_done(void) /* Free unneeded trap tables */ if (!cpu_present(1)) { - free_reserved_page(virt_to_page(&trapbase_cpu1)); + free_reserved_page(virt_to_page(&trapbase_cpu1[0])); } if (!cpu_present(2)) { - free_reserved_page(virt_to_page(&trapbase_cpu2)); + free_reserved_page(virt_to_page(&trapbase_cpu2[0])); } if (!cpu_present(3)) { - free_reserved_page(virt_to_page(&trapbase_cpu3)); + free_reserved_page(virt_to_page(&trapbase_cpu3[0])); } /* Ok, they are spinning and ready to go. */ smp_processors_ready = 1; } -void leon_irq_rotate(int cpu) -{ -} - struct leon_ipi_work { int single; int msk; @@ -354,7 +342,7 @@ static void leon_ipi_resched(int cpu) void leonsmp_ipi_interrupt(void) { - struct leon_ipi_work *work = &__get_cpu_var(leon_ipi_work); + struct leon_ipi_work *work = this_cpu_ptr(&leon_ipi_work); if (work->single) { work->single = 0; @@ -371,7 +359,7 @@ void leonsmp_ipi_interrupt(void) } static struct smp_funcall { - smpfunc_t func; + void *func; unsigned long arg1; unsigned long arg2; unsigned long arg3; @@ -379,12 +367,12 @@ static struct smp_funcall { unsigned long arg5; unsigned long processors_in[NR_CPUS]; /* Set when ipi entered. */ unsigned long processors_out[NR_CPUS]; /* Set when ipi exited. */ -} ccall_info; +} ccall_info __attribute__((aligned(8))); static DEFINE_SPINLOCK(cross_call_lock); /* Cross calls must be serialized, at least currently. */ -static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, +static void leon_cross_call(void *func, cpumask_t mask, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4) { @@ -396,7 +384,7 @@ static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, { /* If you make changes here, make sure gcc generates proper code... */ - register smpfunc_t f asm("i0") = func; + register void *f asm("i0") = func; register unsigned long a1 asm("i1") = arg1; register unsigned long a2 asm("i2") = arg2; register unsigned long a3 asm("i3") = arg3; @@ -456,11 +444,13 @@ static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, /* Running cross calls. */ void leon_cross_call_irq(void) { + void (*func)(unsigned long, unsigned long, unsigned long, unsigned long, + unsigned long) = ccall_info.func; int i = smp_processor_id(); ccall_info.processors_in[i] = 1; - ccall_info.func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, - ccall_info.arg4, ccall_info.arg5); + func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, ccall_info.arg4, + ccall_info.arg5); ccall_info.processors_out[i] = 1; } diff --git a/arch/sparc/kernel/mdesc.c b/arch/sparc/kernel/mdesc.c index 831c001604e8..30f171b7b00c 100644 --- a/arch/sparc/kernel/mdesc.c +++ b/arch/sparc/kernel/mdesc.c @@ -1,25 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 /* mdesc.c: Sun4V machine description handling. * * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> */ #include <linux/kernel.h> #include <linux/types.h> -#include <linux/memblock.h> #include <linux/log2.h> #include <linux/list.h> #include <linux/slab.h> #include <linux/mm.h> #include <linux/miscdevice.h> -#include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/export.h> +#include <linux/refcount.h> #include <asm/cpudata.h> #include <asm/hypervisor.h> #include <asm/mdesc.h> #include <asm/prom.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/oplib.h> #include <asm/smp.h> +#include <asm/adi.h> /* Unlike the OBP device tree, the machine description is a full-on * DAG. An arbitrary number of ARCs are possible from one @@ -37,6 +39,7 @@ struct mdesc_hdr { u32 node_sz; /* node block size */ u32 name_sz; /* name block size */ u32 data_sz; /* data block size */ + char data[]; } __attribute__((aligned(16))); struct mdesc_elem { @@ -70,11 +73,79 @@ struct mdesc_handle { struct list_head list; struct mdesc_mem_ops *mops; void *self_base; - atomic_t refcnt; + refcount_t refcnt; unsigned int handle_size; struct mdesc_hdr mdesc; }; +typedef int (*mdesc_node_info_get_f)(struct mdesc_handle *, u64, + union md_node_info *); +typedef void (*mdesc_node_info_rel_f)(union md_node_info *); +typedef bool (*mdesc_node_match_f)(union md_node_info *, union md_node_info *); + +struct md_node_ops { + char *name; + mdesc_node_info_get_f get_info; + mdesc_node_info_rel_f rel_info; + mdesc_node_match_f node_match; +}; + +static int get_vdev_port_node_info(struct mdesc_handle *md, u64 node, + union md_node_info *node_info); +static void rel_vdev_port_node_info(union md_node_info *node_info); +static bool vdev_port_node_match(union md_node_info *a_node_info, + union md_node_info *b_node_info); + +static int get_ds_port_node_info(struct mdesc_handle *md, u64 node, + union md_node_info *node_info); +static void rel_ds_port_node_info(union md_node_info *node_info); +static bool ds_port_node_match(union md_node_info *a_node_info, + union md_node_info *b_node_info); + +/* supported node types which can be registered */ +static struct md_node_ops md_node_ops_table[] = { + {"virtual-device-port", get_vdev_port_node_info, + rel_vdev_port_node_info, vdev_port_node_match}, + {"domain-services-port", get_ds_port_node_info, + rel_ds_port_node_info, ds_port_node_match}, + {NULL, NULL, NULL, NULL} +}; + +static void mdesc_get_node_ops(const char *node_name, + mdesc_node_info_get_f *get_info_f, + mdesc_node_info_rel_f *rel_info_f, + mdesc_node_match_f *match_f) +{ + int i; + + if (get_info_f) + *get_info_f = NULL; + + if (rel_info_f) + *rel_info_f = NULL; + + if (match_f) + *match_f = NULL; + + if (!node_name) + return; + + for (i = 0; md_node_ops_table[i].name != NULL; i++) { + if (strcmp(md_node_ops_table[i].name, node_name) == 0) { + if (get_info_f) + *get_info_f = md_node_ops_table[i].get_info; + + if (rel_info_f) + *rel_info_f = md_node_ops_table[i].rel_info; + + if (match_f) + *match_f = md_node_ops_table[i].node_match; + + break; + } + } +} + static void mdesc_handle_init(struct mdesc_handle *hp, unsigned int handle_size, void *base) @@ -84,7 +155,7 @@ static void mdesc_handle_init(struct mdesc_handle *hp, memset(hp, 0, handle_size); INIT_LIST_HEAD(&hp->list); hp->self_base = base; - atomic_set(&hp->refcnt, 1); + refcount_set(&hp->refcnt, 1); hp->handle_size = handle_size; } @@ -99,7 +170,7 @@ static struct mdesc_handle * __init mdesc_memblock_alloc(unsigned int mdesc_size mdesc_size); alloc_size = PAGE_ALIGN(handle_size); - paddr = memblock_alloc(alloc_size, PAGE_SIZE); + paddr = memblock_phys_alloc(alloc_size, PAGE_SIZE); hp = NULL; if (paddr) { @@ -114,12 +185,12 @@ static void __init mdesc_memblock_free(struct mdesc_handle *hp) unsigned int alloc_size; unsigned long start; - BUG_ON(atomic_read(&hp->refcnt) != 0); + BUG_ON(refcount_read(&hp->refcnt) != 0); BUG_ON(!list_empty(&hp->list)); alloc_size = PAGE_ALIGN(hp->handle_size); start = __pa(hp); - free_bootmem_late(start, alloc_size); + memblock_free_late(start, alloc_size); } static struct mdesc_mem_ops memblock_mdesc_ops = { @@ -130,31 +201,29 @@ static struct mdesc_mem_ops memblock_mdesc_ops = { static struct mdesc_handle *mdesc_kmalloc(unsigned int mdesc_size) { unsigned int handle_size; + struct mdesc_handle *hp; + unsigned long addr; void *base; handle_size = (sizeof(struct mdesc_handle) - sizeof(struct mdesc_hdr) + mdesc_size); + base = kmalloc(handle_size + 15, GFP_KERNEL | __GFP_RETRY_MAYFAIL); + if (!base) + return NULL; - base = kmalloc(handle_size + 15, GFP_KERNEL | __GFP_NOFAIL); - if (base) { - struct mdesc_handle *hp; - unsigned long addr; - - addr = (unsigned long)base; - addr = (addr + 15UL) & ~15UL; - hp = (struct mdesc_handle *) addr; + addr = (unsigned long)base; + addr = (addr + 15UL) & ~15UL; + hp = (struct mdesc_handle *) addr; - mdesc_handle_init(hp, handle_size, base); - return hp; - } + mdesc_handle_init(hp, handle_size, base); - return NULL; + return hp; } static void mdesc_kfree(struct mdesc_handle *hp) { - BUG_ON(atomic_read(&hp->refcnt) != 0); + BUG_ON(refcount_read(&hp->refcnt) != 0); BUG_ON(!list_empty(&hp->list)); kfree(hp->self_base); @@ -193,7 +262,7 @@ struct mdesc_handle *mdesc_grab(void) spin_lock_irqsave(&mdesc_lock, flags); hp = cur_mdesc; if (hp) - atomic_inc(&hp->refcnt); + refcount_inc(&hp->refcnt); spin_unlock_irqrestore(&mdesc_lock, flags); return hp; @@ -205,7 +274,7 @@ void mdesc_release(struct mdesc_handle *hp) unsigned long flags; spin_lock_irqsave(&mdesc_lock, flags); - if (atomic_dec_and_test(&hp->refcnt)) { + if (refcount_dec_and_test(&hp->refcnt)) { list_del_init(&hp->list); hp->mops->free(hp); } @@ -218,14 +287,31 @@ static struct mdesc_notifier_client *client_list; void mdesc_register_notifier(struct mdesc_notifier_client *client) { + bool supported = false; u64 node; + int i; mutex_lock(&mdesc_mutex); + + /* check to see if the node is supported for registration */ + for (i = 0; md_node_ops_table[i].name != NULL; i++) { + if (strcmp(md_node_ops_table[i].name, client->node_name) == 0) { + supported = true; + break; + } + } + + if (!supported) { + pr_err("MD: %s node not supported\n", client->node_name); + mutex_unlock(&mdesc_mutex); + return; + } + client->next = client_list; client_list = client; mdesc_for_each_node_by_name(cur_mdesc, node, client->node_name) - client->add(cur_mdesc, node); + client->add(cur_mdesc, node, client->node_name); mutex_unlock(&mdesc_mutex); } @@ -249,59 +335,147 @@ static const u64 *parent_cfg_handle(struct mdesc_handle *hp, u64 node) return id; } +static int get_vdev_port_node_info(struct mdesc_handle *md, u64 node, + union md_node_info *node_info) +{ + const u64 *parent_cfg_hdlp; + const char *name; + const u64 *idp; + + /* + * Virtual device nodes are distinguished by: + * 1. "id" property + * 2. "name" property + * 3. parent node "cfg-handle" property + */ + idp = mdesc_get_property(md, node, "id", NULL); + name = mdesc_get_property(md, node, "name", NULL); + parent_cfg_hdlp = parent_cfg_handle(md, node); + + if (!idp || !name || !parent_cfg_hdlp) + return -1; + + node_info->vdev_port.id = *idp; + node_info->vdev_port.name = kstrdup_const(name, GFP_KERNEL); + if (!node_info->vdev_port.name) + return -1; + node_info->vdev_port.parent_cfg_hdl = *parent_cfg_hdlp; + + return 0; +} + +static void rel_vdev_port_node_info(union md_node_info *node_info) +{ + if (node_info && node_info->vdev_port.name) { + kfree_const(node_info->vdev_port.name); + node_info->vdev_port.name = NULL; + } +} + +static bool vdev_port_node_match(union md_node_info *a_node_info, + union md_node_info *b_node_info) +{ + if (a_node_info->vdev_port.id != b_node_info->vdev_port.id) + return false; + + if (a_node_info->vdev_port.parent_cfg_hdl != + b_node_info->vdev_port.parent_cfg_hdl) + return false; + + if (strncmp(a_node_info->vdev_port.name, + b_node_info->vdev_port.name, MDESC_MAX_STR_LEN) != 0) + return false; + + return true; +} + +static int get_ds_port_node_info(struct mdesc_handle *md, u64 node, + union md_node_info *node_info) +{ + const u64 *idp; + + /* DS port nodes use the "id" property to distinguish them */ + idp = mdesc_get_property(md, node, "id", NULL); + if (!idp) + return -1; + + node_info->ds_port.id = *idp; + + return 0; +} + +static void rel_ds_port_node_info(union md_node_info *node_info) +{ +} + +static bool ds_port_node_match(union md_node_info *a_node_info, + union md_node_info *b_node_info) +{ + if (a_node_info->ds_port.id != b_node_info->ds_port.id) + return false; + + return true; +} + /* Run 'func' on nodes which are in A but not in B. */ static void invoke_on_missing(const char *name, struct mdesc_handle *a, struct mdesc_handle *b, - void (*func)(struct mdesc_handle *, u64)) + void (*func)(struct mdesc_handle *, u64, + const char *node_name)) { - u64 node; + mdesc_node_info_get_f get_info_func; + mdesc_node_info_rel_f rel_info_func; + mdesc_node_match_f node_match_func; + union md_node_info a_node_info; + union md_node_info b_node_info; + bool found; + u64 a_node; + u64 b_node; + int rv; + + /* + * Find the get_info, rel_info and node_match ops for the given + * node name + */ + mdesc_get_node_ops(name, &get_info_func, &rel_info_func, + &node_match_func); - mdesc_for_each_node_by_name(a, node, name) { - int found = 0, is_vdc_port = 0; - const char *name_prop; - const u64 *id; - u64 fnode; - - name_prop = mdesc_get_property(a, node, "name", NULL); - if (name_prop && !strcmp(name_prop, "vdc-port")) { - is_vdc_port = 1; - id = parent_cfg_handle(a, node); - } else - id = mdesc_get_property(a, node, "id", NULL); - - if (!id) { - printk(KERN_ERR "MD: Cannot find ID for %s node.\n", - (name_prop ? name_prop : name)); + /* If we didn't find a match, the node type is not supported */ + if (!get_info_func || !rel_info_func || !node_match_func) { + pr_err("MD: %s node type is not supported\n", name); + return; + } + + mdesc_for_each_node_by_name(a, a_node, name) { + found = false; + + rv = get_info_func(a, a_node, &a_node_info); + if (rv != 0) { + pr_err("MD: Cannot find 1 or more required match properties for %s node.\n", + name); continue; } - mdesc_for_each_node_by_name(b, fnode, name) { - const u64 *fid; - - if (is_vdc_port) { - name_prop = mdesc_get_property(b, fnode, - "name", NULL); - if (!name_prop || - strcmp(name_prop, "vdc-port")) - continue; - fid = parent_cfg_handle(b, fnode); - if (!fid) { - printk(KERN_ERR "MD: Cannot find ID " - "for vdc-port node.\n"); - continue; - } - } else - fid = mdesc_get_property(b, fnode, - "id", NULL); - - if (*id == *fid) { - found = 1; + /* Check each node in B for node matching a_node */ + mdesc_for_each_node_by_name(b, b_node, name) { + rv = get_info_func(b, b_node, &b_node_info); + if (rv != 0) + continue; + + if (node_match_func(&a_node_info, &b_node_info)) { + found = true; + rel_info_func(&b_node_info); break; } + + rel_info_func(&b_node_info); } + + rel_info_func(&a_node_info); + if (!found) - func(a, node); + func(a, a_node, name); } } @@ -344,7 +518,7 @@ void mdesc_update(void) if (status != HV_EOK || real_len > len) { printk(KERN_ERR "MD: mdesc reread fails with %lu\n", status); - atomic_dec(&hp->refcnt); + refcount_dec(&hp->refcnt); mdesc_free(hp); goto out; } @@ -357,7 +531,7 @@ void mdesc_update(void) mdesc_notify_clients(orig_hp, hp); spin_lock_irqsave(&mdesc_lock, flags); - if (atomic_dec_and_test(&orig_hp->refcnt)) + if (refcount_dec_and_test(&orig_hp->refcnt)) mdesc_free(orig_hp); else list_add(&orig_hp->list, &mdesc_zombie_list); @@ -367,9 +541,79 @@ out: mutex_unlock(&mdesc_mutex); } +u64 mdesc_get_node(struct mdesc_handle *hp, const char *node_name, + union md_node_info *node_info) +{ + mdesc_node_info_get_f get_info_func; + mdesc_node_info_rel_f rel_info_func; + mdesc_node_match_f node_match_func; + union md_node_info hp_node_info; + u64 hp_node; + int rv; + + if (hp == NULL || node_name == NULL || node_info == NULL) + return MDESC_NODE_NULL; + + /* Find the ops for the given node name */ + mdesc_get_node_ops(node_name, &get_info_func, &rel_info_func, + &node_match_func); + + /* If we didn't find ops for the given node name, it is not supported */ + if (!get_info_func || !rel_info_func || !node_match_func) { + pr_err("MD: %s node is not supported\n", node_name); + return -EINVAL; + } + + mdesc_for_each_node_by_name(hp, hp_node, node_name) { + rv = get_info_func(hp, hp_node, &hp_node_info); + if (rv != 0) + continue; + + if (node_match_func(node_info, &hp_node_info)) + break; + + rel_info_func(&hp_node_info); + } + + rel_info_func(&hp_node_info); + + return hp_node; +} +EXPORT_SYMBOL(mdesc_get_node); + +int mdesc_get_node_info(struct mdesc_handle *hp, u64 node, + const char *node_name, union md_node_info *node_info) +{ + mdesc_node_info_get_f get_info_func; + int rv; + + if (hp == NULL || node == MDESC_NODE_NULL || + node_name == NULL || node_info == NULL) + return -EINVAL; + + /* Find the get_info op for the given node name */ + mdesc_get_node_ops(node_name, &get_info_func, NULL, NULL); + + /* If we didn't find a get_info_func, the node name is not supported */ + if (get_info_func == NULL) { + pr_err("MD: %s node is not supported\n", node_name); + return -EINVAL; + } + + rv = get_info_func(hp, node, node_info); + if (rv != 0) { + pr_err("MD: Cannot find 1 or more required match properties for %s node.\n", + node_name); + return -1; + } + + return 0; +} +EXPORT_SYMBOL(mdesc_get_node_info); + static struct mdesc_elem *node_block(struct mdesc_hdr *mdesc) { - return (struct mdesc_elem *) (mdesc + 1); + return (struct mdesc_elem *) mdesc->data; } static void *name_block(struct mdesc_hdr *mdesc) @@ -571,9 +815,7 @@ static void __init report_platform_properties(void) mdesc_release(hp); } -static void __cpuinit fill_in_one_cache(cpuinfo_sparc *c, - struct mdesc_handle *hp, - u64 mp) +static void fill_in_one_cache(cpuinfo_sparc *c, struct mdesc_handle *hp, u64 mp) { const u64 *level = mdesc_get_property(hp, mp, "level", NULL); const u64 *size = mdesc_get_property(hp, mp, "size", NULL); @@ -616,45 +858,76 @@ static void __cpuinit fill_in_one_cache(cpuinfo_sparc *c, } } -static void __cpuinit mark_core_ids(struct mdesc_handle *hp, u64 mp, int core_id) +static void find_back_node_value(struct mdesc_handle *hp, u64 node, + char *srch_val, + void (*func)(struct mdesc_handle *, u64, int), + u64 val, int depth) { - u64 a; + u64 arc; - mdesc_for_each_arc(a, hp, mp, MDESC_ARC_TYPE_BACK) { - u64 t = mdesc_arc_target(hp, a); - const char *name; - const u64 *id; + /* Since we have an estimate of recursion depth, do a sanity check. */ + if (depth == 0) + return; - name = mdesc_node_name(hp, t); - if (!strcmp(name, "cpu")) { - id = mdesc_get_property(hp, t, "id", NULL); - if (*id < NR_CPUS) - cpu_data(*id).core_id = core_id; - } else { - u64 j; + mdesc_for_each_arc(arc, hp, node, MDESC_ARC_TYPE_BACK) { + u64 n = mdesc_arc_target(hp, arc); + const char *name = mdesc_node_name(hp, n); - mdesc_for_each_arc(j, hp, t, MDESC_ARC_TYPE_BACK) { - u64 n = mdesc_arc_target(hp, j); - const char *n_name; + if (!strcmp(srch_val, name)) + (*func)(hp, n, val); - n_name = mdesc_node_name(hp, n); - if (strcmp(n_name, "cpu")) - continue; + find_back_node_value(hp, n, srch_val, func, val, depth-1); + } +} - id = mdesc_get_property(hp, n, "id", NULL); - if (*id < NR_CPUS) - cpu_data(*id).core_id = core_id; - } - } +static void __mark_core_id(struct mdesc_handle *hp, u64 node, + int core_id) +{ + const u64 *id = mdesc_get_property(hp, node, "id", NULL); + + if (*id < num_possible_cpus()) + cpu_data(*id).core_id = core_id; +} + +static void __mark_max_cache_id(struct mdesc_handle *hp, u64 node, + int max_cache_id) +{ + const u64 *id = mdesc_get_property(hp, node, "id", NULL); + + if (*id < num_possible_cpus()) { + cpu_data(*id).max_cache_id = max_cache_id; + + /** + * On systems without explicit socket descriptions socket + * is max_cache_id + */ + cpu_data(*id).sock_id = max_cache_id; } } -static void __cpuinit set_core_ids(struct mdesc_handle *hp) +static void mark_core_ids(struct mdesc_handle *hp, u64 mp, + int core_id) +{ + find_back_node_value(hp, mp, "cpu", __mark_core_id, core_id, 10); +} + +static void mark_max_cache_ids(struct mdesc_handle *hp, u64 mp, + int max_cache_id) +{ + find_back_node_value(hp, mp, "cpu", __mark_max_cache_id, + max_cache_id, 10); +} + +static void set_core_ids(struct mdesc_handle *hp) { int idx; u64 mp; idx = 1; + + /* Identify unique cores by looking for cpus backpointed to by + * level 1 instruction caches. + */ mdesc_for_each_node_by_name(hp, mp, "cache") { const u64 *level; const char *type; @@ -669,12 +942,75 @@ static void __cpuinit set_core_ids(struct mdesc_handle *hp) continue; mark_core_ids(hp, mp, idx); + idx++; + } +} +static int set_max_cache_ids_by_cache(struct mdesc_handle *hp, int level) +{ + u64 mp; + int idx = 1; + int fnd = 0; + + /** + * Identify unique highest level of shared cache by looking for cpus + * backpointed to by shared level N caches. + */ + mdesc_for_each_node_by_name(hp, mp, "cache") { + const u64 *cur_lvl; + + cur_lvl = mdesc_get_property(hp, mp, "level", NULL); + if (*cur_lvl != level) + continue; + mark_max_cache_ids(hp, mp, idx); idx++; + fnd = 1; } + return fnd; } -static void __cpuinit mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id) +static void set_sock_ids_by_socket(struct mdesc_handle *hp, u64 mp) +{ + int idx = 1; + + mdesc_for_each_node_by_name(hp, mp, "socket") { + u64 a; + + mdesc_for_each_arc(a, hp, mp, MDESC_ARC_TYPE_FWD) { + u64 t = mdesc_arc_target(hp, a); + const char *name; + const u64 *id; + + name = mdesc_node_name(hp, t); + if (strcmp(name, "cpu")) + continue; + + id = mdesc_get_property(hp, t, "id", NULL); + if (*id < num_possible_cpus()) + cpu_data(*id).sock_id = idx; + } + idx++; + } +} + +static void set_sock_ids(struct mdesc_handle *hp) +{ + u64 mp; + + /** + * Find the highest level of shared cache which pre-T7 is also + * the socket. + */ + if (!set_max_cache_ids_by_cache(hp, 3)) + set_max_cache_ids_by_cache(hp, 2); + + /* If machine description exposes sockets data use it.*/ + mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "sockets"); + if (mp != MDESC_NODE_NULL) + set_sock_ids_by_socket(hp, mp); +} + +static void mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id) { u64 a; @@ -693,7 +1029,7 @@ static void __cpuinit mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id } } -static void __cpuinit __set_proc_ids(struct mdesc_handle *hp, const char *exec_unit_name) +static void __set_proc_ids(struct mdesc_handle *hp, const char *exec_unit_name) { int idx; u64 mp; @@ -709,19 +1045,18 @@ static void __cpuinit __set_proc_ids(struct mdesc_handle *hp, const char *exec_u continue; mark_proc_ids(hp, mp, idx); - idx++; } } -static void __cpuinit set_proc_ids(struct mdesc_handle *hp) +static void set_proc_ids(struct mdesc_handle *hp) { __set_proc_ids(hp, "exec_unit"); __set_proc_ids(hp, "exec-unit"); } -static void __cpuinit get_one_mondo_bits(const u64 *p, unsigned int *mask, - unsigned long def, unsigned long max) +static void get_one_mondo_bits(const u64 *p, unsigned int *mask, + unsigned long def, unsigned long max) { u64 val; @@ -742,8 +1077,8 @@ use_default: *mask = ((1U << def) * 64U) - 1U; } -static void __cpuinit get_mondo_data(struct mdesc_handle *hp, u64 mp, - struct trap_per_cpu *tb) +static void get_mondo_data(struct mdesc_handle *hp, u64 mp, + struct trap_per_cpu *tb) { static int printed; const u64 *val; @@ -769,7 +1104,7 @@ static void __cpuinit get_mondo_data(struct mdesc_handle *hp, u64 mp, } } -static void * __cpuinit mdesc_iterate_over_cpus(void *(*func)(struct mdesc_handle *, u64, int, void *), void *arg, cpumask_t *mask) +static void *mdesc_iterate_over_cpus(void *(*func)(struct mdesc_handle *, u64, int, void *), void *arg, cpumask_t *mask) { struct mdesc_handle *hp = mdesc_grab(); void *ret = NULL; @@ -799,7 +1134,8 @@ out: return ret; } -static void * __cpuinit record_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) +static void *record_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, + void *arg) { ncpus_probed++; #ifdef CONFIG_SMP @@ -808,7 +1144,7 @@ static void * __cpuinit record_one_cpu(struct mdesc_handle *hp, u64 mp, int cpui return NULL; } -void __cpuinit mdesc_populate_present_mask(cpumask_t *mask) +void mdesc_populate_present_mask(cpumask_t *mask) { if (tlb_type != hypervisor) return; @@ -841,7 +1177,8 @@ void __init mdesc_get_page_sizes(cpumask_t *mask, unsigned long *pgsz_mask) mdesc_iterate_over_cpus(check_one_pgsz, pgsz_mask, mask); } -static void * __cpuinit fill_in_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) +static void *fill_in_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, + void *arg) { const u64 *cfreq = mdesc_get_property(hp, mp, "clock-frequency", NULL); struct trap_per_cpu *tb; @@ -890,49 +1227,86 @@ static void * __cpuinit fill_in_one_cpu(struct mdesc_handle *hp, u64 mp, int cpu return NULL; } -void __cpuinit mdesc_fill_in_cpu_data(cpumask_t *mask) +void mdesc_fill_in_cpu_data(cpumask_t *mask) { struct mdesc_handle *hp; mdesc_iterate_over_cpus(fill_in_one_cpu, NULL, mask); -#ifdef CONFIG_SMP - sparc64_multi_core = 1; -#endif - hp = mdesc_grab(); set_core_ids(hp); set_proc_ids(hp); + set_sock_ids(hp); mdesc_release(hp); smp_fill_in_sib_core_maps(); } -static ssize_t mdesc_read(struct file *file, char __user *buf, - size_t len, loff_t *offp) +/* mdesc_open() - Grab a reference to mdesc_handle when /dev/mdesc is + * opened. Hold this reference until /dev/mdesc is closed to ensure + * mdesc data structure is not released underneath us. Store the + * pointer to mdesc structure in private_data for read and seek to use + */ +static int mdesc_open(struct inode *inode, struct file *file) { struct mdesc_handle *hp = mdesc_grab(); - int err; if (!hp) return -ENODEV; - err = hp->handle_size; - if (len < hp->handle_size) - err = -EMSGSIZE; - else if (copy_to_user(buf, &hp->mdesc, hp->handle_size)) - err = -EFAULT; - mdesc_release(hp); + file->private_data = hp; + + return 0; +} + +static ssize_t mdesc_read(struct file *file, char __user *buf, + size_t len, loff_t *offp) +{ + struct mdesc_handle *hp = file->private_data; + unsigned char *mdesc; + int bytes_left, count = len; + + if (*offp >= hp->handle_size) + return 0; + + bytes_left = hp->handle_size - *offp; + if (count > bytes_left) + count = bytes_left; + + mdesc = (unsigned char *)&hp->mdesc; + mdesc += *offp; + if (!copy_to_user(buf, mdesc, count)) { + *offp += count; + return count; + } else { + return -EFAULT; + } +} - return err; +static loff_t mdesc_llseek(struct file *file, loff_t offset, int whence) +{ + struct mdesc_handle *hp = file->private_data; + + return no_seek_end_llseek_size(file, offset, whence, hp->handle_size); +} + +/* mdesc_close() - /dev/mdesc is being closed, release the reference to + * mdesc structure. + */ +static int mdesc_close(struct inode *inode, struct file *file) +{ + mdesc_release(file->private_data); + return 0; } static const struct file_operations mdesc_fops = { - .read = mdesc_read, - .owner = THIS_MODULE, - .llseek = noop_llseek, + .open = mdesc_open, + .read = mdesc_read, + .llseek = mdesc_llseek, + .release = mdesc_close, + .owner = THIS_MODULE, }; static struct miscdevice mdesc_misc = { @@ -974,5 +1348,6 @@ void __init sun4v_mdesc_init(void) cur_mdesc = hp; + mdesc_adi_init(); report_platform_properties(); } diff --git a/arch/sparc/kernel/misctrap.S b/arch/sparc/kernel/misctrap.S index 753b4f031bfb..b5c84177521e 100644 --- a/arch/sparc/kernel/misctrap.S +++ b/arch/sparc/kernel/misctrap.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifdef CONFIG_KGDB .globl arch_kgdb_breakpoint .type arch_kgdb_breakpoint,#function @@ -18,8 +19,7 @@ __do_privact: 109: or %g7, %lo(109b), %g7 call do_privact add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size __do_privact,.-__do_privact .type do_mna,#function @@ -46,8 +46,7 @@ do_mna: mov %l5, %o2 call mem_address_unaligned add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size do_mna,.-do_mna .type do_lddfmna,#function @@ -65,8 +64,7 @@ do_lddfmna: mov %l5, %o2 call handle_lddfmna add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size do_lddfmna,.-do_lddfmna .type do_stdfmna,#function @@ -84,7 +82,7 @@ do_stdfmna: mov %l5, %o2 call handle_stdfmna add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap + ba,a,pt %xcc, rtrap nop .size do_stdfmna,.-do_stdfmna diff --git a/arch/sparc/kernel/module.c b/arch/sparc/kernel/module.c index 4435488ebe25..49740450a685 100644 --- a/arch/sparc/kernel/module.c +++ b/arch/sparc/kernel/module.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* Kernel module help for sparc64. * * Copyright (C) 2001 Rusty Russell. @@ -20,36 +21,6 @@ #include "entry.h" -#ifdef CONFIG_SPARC64 - -#include <linux/jump_label.h> - -static void *module_map(unsigned long size) -{ - if (PAGE_ALIGN(size) > MODULES_LEN) - return NULL; - return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL, -1, - __builtin_return_address(0)); -} -#else -static void *module_map(unsigned long size) -{ - return vmalloc(size); -} -#endif /* CONFIG_SPARC64 */ - -void *module_alloc(unsigned long size) -{ - void *ret; - - ret = module_map(size); - if (ret) - memset(ret, 0, size); - - return ret; -} - /* Make generic code ignore STT_REGISTER dummy undefined symbols. */ int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, @@ -116,6 +87,7 @@ int apply_relocate_add(Elf_Shdr *sechdrs, break; #ifdef CONFIG_SPARC64 case R_SPARC_64: + case R_SPARC_UA64: location[0] = v >> 56; location[1] = v >> 48; location[2] = v >> 40; @@ -170,7 +142,7 @@ int apply_relocate_add(Elf_Shdr *sechdrs, break; default: - printk(KERN_ERR "module %s: Unknown relocation: %x\n", + printk(KERN_ERR "module %s: Unknown relocation: 0x%x\n", me->name, (int) (ELF_R_TYPE(rel[i].r_info) & 0xff)); return -ENOEXEC; @@ -207,9 +179,6 @@ int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *me) { - /* make jump label nops */ - jump_label_apply_nops(me); - do_patch_sections(hdr, sechdrs); /* Cheetah's I-cache is fully coherent. */ diff --git a/arch/sparc/kernel/nmi.c b/arch/sparc/kernel/nmi.c index 6479256fd5a4..149adc094753 100644 --- a/arch/sparc/kernel/nmi.c +++ b/arch/sparc/kernel/nmi.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* Pseudo NMI support on sparc64 systems. * * Copyright (C) 2009 David S. Miller <davem@davemloft.net> @@ -42,7 +43,7 @@ static int panic_on_timeout; */ atomic_t nmi_active = ATOMIC_INIT(0); /* oprofile uses this */ EXPORT_SYMBOL(nmi_active); - +static int nmi_init_done; static unsigned int nmi_hz = HZ; static DEFINE_PER_CPU(short, wd_enabled); static int endflag __initdata; @@ -51,7 +52,7 @@ static DEFINE_PER_CPU(unsigned int, last_irq_sum); static DEFINE_PER_CPU(long, alert_counter); static DEFINE_PER_CPU(int, nmi_touch); -void touch_nmi_watchdog(void) +void arch_touch_nmi_watchdog(void) { if (atomic_read(&nmi_active)) { int cpu; @@ -61,34 +62,26 @@ void touch_nmi_watchdog(void) per_cpu(nmi_touch, cpu) = 1; } } +} +EXPORT_SYMBOL(arch_touch_nmi_watchdog); - touch_softlockup_watchdog(); +int __init watchdog_hardlockup_probe(void) +{ + return 0; } -EXPORT_SYMBOL(touch_nmi_watchdog); static void die_nmi(const char *str, struct pt_regs *regs, int do_panic) { + int this_cpu = smp_processor_id(); + if (notify_die(DIE_NMIWATCHDOG, str, regs, 0, pt_regs_trap_type(regs), SIGINT) == NOTIFY_STOP) return; - console_verbose(); - bust_spinlocks(1); - - printk(KERN_EMERG "%s", str); - printk(" on CPU%d, ip %08lx, registers:\n", - smp_processor_id(), regs->tpc); - show_regs(regs); - dump_stack(); - - bust_spinlocks(0); - if (do_panic || panic_on_oops) - panic("Non maskable interrupt"); - - nmi_exit(); - local_irq_enable(); - do_exit(SIGBUS); + panic("Watchdog detected hard LOCKUP on cpu %d", this_cpu); + else + WARN(1, "Watchdog detected hard LOCKUP on cpu %d", this_cpu); } notrace __kprobes void perfctr_irq(int irq, struct pt_regs *regs) @@ -111,20 +104,20 @@ notrace __kprobes void perfctr_irq(int irq, struct pt_regs *regs) pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_disable); sum = local_cpu_data().irq0_irqs; - if (__get_cpu_var(nmi_touch)) { - __get_cpu_var(nmi_touch) = 0; + if (__this_cpu_read(nmi_touch)) { + __this_cpu_write(nmi_touch, 0); touched = 1; } - if (!touched && __get_cpu_var(last_irq_sum) == sum) { + if (!touched && __this_cpu_read(last_irq_sum) == sum) { __this_cpu_inc(alert_counter); if (__this_cpu_read(alert_counter) == 30 * nmi_hz) die_nmi("BUG: NMI Watchdog detected LOCKUP", regs, panic_on_timeout); } else { - __get_cpu_var(last_irq_sum) = sum; + __this_cpu_write(last_irq_sum, sum); __this_cpu_write(alert_counter, 0); } - if (__get_cpu_var(wd_enabled)) { + if (__this_cpu_read(wd_enabled)) { pcr_ops->write_pic(0, pcr_ops->nmi_picl_value(nmi_hz)); pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_enable); } @@ -141,7 +134,6 @@ static inline unsigned int get_nmi_count(int cpu) static __init void nmi_cpu_busy(void *data) { - local_irq_enable_in_hardirq(); while (endflag == 0) mb(); } @@ -165,8 +157,10 @@ static void report_broken_nmi(int cpu, int *prev_nmi_count) void stop_nmi_watchdog(void *unused) { + if (!__this_cpu_read(wd_enabled)) + return; pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_disable); - __get_cpu_var(wd_enabled) = 0; + __this_cpu_write(wd_enabled, 0); atomic_dec(&nmi_active); } @@ -178,7 +172,8 @@ static int __init check_nmi_watchdog(void) if (!atomic_read(&nmi_active)) return 0; - prev_nmi_count = kmalloc(nr_cpu_ids * sizeof(unsigned int), GFP_KERNEL); + prev_nmi_count = kmalloc_array(nr_cpu_ids, sizeof(unsigned int), + GFP_KERNEL); if (!prev_nmi_count) { err = -ENOMEM; goto error; @@ -219,7 +214,10 @@ error: void start_nmi_watchdog(void *unused) { - __get_cpu_var(wd_enabled) = 1; + if (__this_cpu_read(wd_enabled)) + return; + + __this_cpu_write(wd_enabled, 1); atomic_inc(&nmi_active); pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_disable); @@ -230,7 +228,7 @@ void start_nmi_watchdog(void *unused) static void nmi_adjust_hz_one(void *unused) { - if (!__get_cpu_var(wd_enabled)) + if (!__this_cpu_read(wd_enabled)) return; pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_disable); @@ -271,6 +269,8 @@ int __init nmi_init(void) } } + nmi_init_done = 1; + return err; } @@ -279,6 +279,39 @@ static int __init setup_nmi_watchdog(char *str) if (!strncmp(str, "panic", 5)) panic_on_timeout = 1; - return 0; + return 1; } __setup("nmi_watchdog=", setup_nmi_watchdog); + +/* + * sparc specific NMI watchdog enable function. + * Enables watchdog if it is not enabled already. + */ +void watchdog_hardlockup_enable(unsigned int cpu) +{ + if (atomic_read(&nmi_active) == -1) { + pr_warn("NMI watchdog cannot be enabled or disabled\n"); + return; + } + + /* + * watchdog thread could start even before nmi_init is called. + * Just Return in that case. Let nmi_init finish the init + * process first. + */ + if (!nmi_init_done) + return; + + smp_call_function_single(cpu, start_nmi_watchdog, NULL, 1); +} +/* + * sparc specific NMI watchdog disable function. + * Disables watchdog if it is not disabled already. + */ +void watchdog_hardlockup_disable(unsigned int cpu) +{ + if (atomic_read(&nmi_active) == -1) + pr_warn_once("NMI watchdog cannot be enabled or disabled\n"); + else + smp_call_function_single(cpu, stop_nmi_watchdog, NULL, 1); +} diff --git a/arch/sparc/kernel/of_device_32.c b/arch/sparc/kernel/of_device_32.c index 185aa96fa5be..284a4cafa432 100644 --- a/arch/sparc/kernel/of_device_32.c +++ b/arch/sparc/kernel/of_device_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/string.h> #include <linux/kernel.h> #include <linux/of.h> @@ -6,8 +7,9 @@ #include <linux/slab.h> #include <linux/errno.h> #include <linux/irq.h> -#include <linux/of_device.h> #include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/dma-mapping.h> #include <asm/leon.h> #include <asm/leon_amba.h> @@ -20,14 +22,14 @@ static int of_bus_pci_match(struct device_node *np) { - if (!strcmp(np->type, "pci") || !strcmp(np->type, "pciex")) { + if (of_node_is_type(np, "pci") || of_node_is_type(np, "pciex")) { /* Do not do PCI specific frobbing if the * PCI bridge lacks a ranges property. We * want to pass it through up to the next * parent as-is, not with the PCI translate * method which chops off the top address cell. */ - if (!of_find_property(np, "ranges", NULL)) + if (!of_property_present(np, "ranges")) return 0; return 1; @@ -105,7 +107,7 @@ static unsigned long of_bus_sbus_get_flags(const u32 *addr, unsigned long flags) static int of_bus_ambapp_match(struct device_node *np) { - return !strcmp(np->type, "ambapp"); + return of_node_is_type(np, "ambapp"); } static void of_bus_ambapp_count_cells(struct device_node *child, @@ -221,7 +223,7 @@ static int __init build_one_resource(struct device_node *parent, static int __init use_1to1_mapping(struct device_node *pp) { /* If we have a ranges property in the parent, use it. */ - if (of_find_property(pp, "ranges", NULL) != NULL) + if (of_property_present(pp, "ranges")) return 0; /* Some SBUS devices use intermediate nodes to express @@ -230,10 +232,10 @@ static int __init use_1to1_mapping(struct device_node *pp) * But, we should still pass the translation work up * to the SBUS itself. */ - if (!strcmp(pp->name, "dma") || - !strcmp(pp->name, "espdma") || - !strcmp(pp->name, "ledma") || - !strcmp(pp->name, "lebuffer")) + if (of_node_name_eq(pp, "dma") || + of_node_name_eq(pp, "espdma") || + of_node_name_eq(pp, "ledma") || + of_node_name_eq(pp, "lebuffer")) return 0; return 1; @@ -322,8 +324,8 @@ static void __init build_device_resources(struct platform_device *op, memset(r, 0, sizeof(*r)); if (of_resource_verbose) - printk("%s reg[%d] -> %llx\n", - op->dev.of_node->full_name, index, + printk("%pOF reg[%d] -> %llx\n", + op->dev.of_node, index, result); if (result != OF_BAD_ADDR) { @@ -331,7 +333,7 @@ static void __init build_device_resources(struct platform_device *op, r->end = result + size - 1; r->flags = flags | ((result >> 32ULL) & 0xffUL); } - r->name = op->dev.of_node->name; + r->name = op->dev.of_node->full_name; } } @@ -380,9 +382,12 @@ static struct platform_device * __init scan_one_device(struct device_node *dp, else dev_set_name(&op->dev, "%08x", dp->phandle); + op->dev.coherent_dma_mask = DMA_BIT_MASK(32); + op->dev.dma_mask = &op->dev.coherent_dma_mask; + if (of_device_register(op)) { - printk("%s: Could not register of device.\n", - dp->full_name); + printk("%pOF: Could not register of device.\n", dp); + put_device(&op->dev); kfree(op); op = NULL; } diff --git a/arch/sparc/kernel/of_device_64.c b/arch/sparc/kernel/of_device_64.c index 7bbdc26d9512..f53092b07b9e 100644 --- a/arch/sparc/kernel/of_device_64.c +++ b/arch/sparc/kernel/of_device_64.c @@ -1,14 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/string.h> #include <linux/kernel.h> -#include <linux/of.h> +#include <linux/dma-mapping.h> #include <linux/init.h> #include <linux/export.h> #include <linux/mod_devicetable.h> #include <linux/slab.h> #include <linux/errno.h> #include <linux/irq.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_platform.h> +#include <linux/platform_device.h> #include <asm/spitfire.h> #include "of_device_common.h" @@ -44,7 +46,7 @@ EXPORT_SYMBOL(of_iounmap); static int of_bus_pci_match(struct device_node *np) { - if (!strcmp(np->name, "pci")) { + if (of_node_name_eq(np, "pci")) { const char *model = of_get_property(np, "model", NULL); if (model && !strcmp(model, "SUNW,simba")) @@ -56,7 +58,7 @@ static int of_bus_pci_match(struct device_node *np) * parent as-is, not with the PCI translate * method which chops off the top address cell. */ - if (!of_find_property(np, "ranges", NULL)) + if (!of_property_present(np, "ranges")) return 0; return 1; @@ -75,8 +77,8 @@ static int of_bus_simba_match(struct device_node *np) /* Treat PCI busses lacking ranges property just like * simba. */ - if (!strcmp(np->name, "pci")) { - if (!of_find_property(np, "ranges", NULL)) + if (of_node_name_eq(np, "pci")) { + if (!of_property_present(np, "ranges")) return 1; } @@ -168,8 +170,8 @@ static unsigned long of_bus_pci_get_flags(const u32 *addr, unsigned long flags) */ static int of_bus_fhc_match(struct device_node *np) { - return !strcmp(np->name, "fhc") || - !strcmp(np->name, "central"); + return of_node_name_eq(np, "fhc") || + of_node_name_eq(np, "central"); } #define of_bus_fhc_count_cells of_bus_sbus_count_cells @@ -281,7 +283,7 @@ static int __init build_one_resource(struct device_node *parent, static int __init use_1to1_mapping(struct device_node *pp) { /* If we have a ranges property in the parent, use it. */ - if (of_find_property(pp, "ranges", NULL) != NULL) + if (of_property_present(pp, "ranges")) return 0; /* If the parent is the dma node of an ISA bus, pass @@ -293,17 +295,17 @@ static int __init use_1to1_mapping(struct device_node *pp) * But, we should still pass the translation work up * to the SBUS itself. */ - if (!strcmp(pp->name, "dma") || - !strcmp(pp->name, "espdma") || - !strcmp(pp->name, "ledma") || - !strcmp(pp->name, "lebuffer")) + if (of_node_name_eq(pp, "dma") || + of_node_name_eq(pp, "espdma") || + of_node_name_eq(pp, "ledma") || + of_node_name_eq(pp, "lebuffer")) return 0; /* Similarly for all PCI bridges, if we get this far * it lacks a ranges property, and this will include * cases like Simba. */ - if (!strcmp(pp->name, "pci")) + if (of_node_name_eq(pp, "pci")) return 0; return 1; @@ -339,9 +341,9 @@ static void __init build_device_resources(struct platform_device *op, /* Prevent overrunning the op->resources[] array. */ if (num_reg > PROMREG_MAX) { - printk(KERN_WARNING "%s: Too many regs (%d), " + printk(KERN_WARNING "%pOF: Too many regs (%d), " "limiting to %d.\n", - op->dev.of_node->full_name, num_reg, PROMREG_MAX); + op->dev.of_node, num_reg, PROMREG_MAX); num_reg = PROMREG_MAX; } @@ -399,8 +401,8 @@ static void __init build_device_resources(struct platform_device *op, memset(r, 0, sizeof(*r)); if (of_resource_verbose) - printk("%s reg[%d] -> %llx\n", - op->dev.of_node->full_name, index, + printk("%pOF reg[%d] -> %llx\n", + op->dev.of_node, index, result); if (result != OF_BAD_ADDR) { @@ -411,7 +413,7 @@ static void __init build_device_resources(struct platform_device *op, r->end = result + size - 1; r->flags = flags; } - r->name = op->dev.of_node->name; + r->name = op->dev.of_node->full_name; } } @@ -546,8 +548,8 @@ static unsigned int __init build_one_device_irq(struct platform_device *op, dp->irq_trans->data); if (of_irq_verbose) - printk("%s: direct translate %x --> %x\n", - dp->full_name, orig_irq, irq); + printk("%pOF: direct translate %x --> %x\n", + dp, orig_irq, irq); goto out; } @@ -558,7 +560,7 @@ static unsigned int __init build_one_device_irq(struct platform_device *op, * * If we hit a bus type or situation we cannot handle, we * stop and assume that the original IRQ number was in a - * format which has special meaning to it's immediate parent. + * format which has special meaning to its immediate parent. */ pp = dp->parent; ip = NULL; @@ -577,10 +579,9 @@ static unsigned int __init build_one_device_irq(struct platform_device *op, &irq); if (of_irq_verbose) - printk("%s: Apply [%s:%x] imap --> [%s:%x]\n", - op->dev.of_node->full_name, - pp->full_name, this_orig_irq, - of_node_full_name(iret), irq); + printk("%pOF: Apply [%pOF:%x] imap --> [%pOF:%x]\n", + op->dev.of_node, + pp, this_orig_irq, iret, irq); if (!iret) break; @@ -590,15 +591,15 @@ static unsigned int __init build_one_device_irq(struct platform_device *op, break; } } else { - if (!strcmp(pp->name, "pci")) { + if (of_node_name_eq(pp, "pci")) { unsigned int this_orig_irq = irq; irq = pci_irq_swizzle(dp, pp, irq); if (of_irq_verbose) - printk("%s: PCI swizzle [%s] " + printk("%pOF: PCI swizzle [%pOF] " "%x --> %x\n", - op->dev.of_node->full_name, - pp->full_name, this_orig_irq, + op->dev.of_node, + pp, this_orig_irq, irq); } @@ -617,16 +618,13 @@ static unsigned int __init build_one_device_irq(struct platform_device *op, irq = ip->irq_trans->irq_build(op->dev.of_node, irq, ip->irq_trans->data); if (of_irq_verbose) - printk("%s: Apply IRQ trans [%s] %x --> %x\n", - op->dev.of_node->full_name, ip->full_name, orig_irq, irq); + printk("%pOF: Apply IRQ trans [%pOF] %x --> %x\n", + op->dev.of_node, ip, orig_irq, irq); out: nid = of_node_to_nid(dp); if (nid != -1) { - cpumask_t numa_mask; - - cpumask_copy(&numa_mask, cpumask_of_node(nid)); - irq_set_affinity(irq, &numa_mask); + irq_set_affinity(irq, cpumask_of_node(nid)); } return irq; @@ -654,9 +652,9 @@ static struct platform_device * __init scan_one_device(struct device_node *dp, /* Prevent overrunning the op->irqs[] array. */ if (op->archdata.num_irqs > PROMINTR_MAX) { - printk(KERN_WARNING "%s: Too many irqs (%d), " + printk(KERN_WARNING "%pOF: Too many irqs (%d), " "limiting to %d.\n", - dp->full_name, op->archdata.num_irqs, PROMINTR_MAX); + dp, op->archdata.num_irqs, PROMINTR_MAX); op->archdata.num_irqs = PROMINTR_MAX; } memcpy(op->archdata.irqs, irq, op->archdata.num_irqs * 4); @@ -674,10 +672,12 @@ static struct platform_device * __init scan_one_device(struct device_node *dp, dev_set_name(&op->dev, "root"); else dev_set_name(&op->dev, "%08x", dp->phandle); + op->dev.coherent_dma_mask = DMA_BIT_MASK(32); + op->dev.dma_mask = &op->dev.coherent_dma_mask; if (of_device_register(op)) { - printk("%s: Could not register of device.\n", - dp->full_name); + printk("%pOF: Could not register of device.\n", dp); + put_device(&op->dev); kfree(op); op = NULL; } diff --git a/arch/sparc/kernel/of_device_common.c b/arch/sparc/kernel/of_device_common.c index de199bf0cb05..ba2a6ae23508 100644 --- a/arch/sparc/kernel/of_device_common.c +++ b/arch/sparc/kernel/of_device_common.c @@ -1,13 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only #include <linux/string.h> #include <linux/kernel.h> -#include <linux/of.h> -#include <linux/init.h> #include <linux/export.h> #include <linux/mod_devicetable.h> #include <linux/errno.h> #include <linux/irq.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_platform.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> #include "of_device_common.h" @@ -65,6 +67,7 @@ void of_propagate_archdata(struct platform_device *bus) op->dev.archdata.stc = bus_sd->stc; op->dev.archdata.host_controller = bus_sd->host_controller; op->dev.archdata.numa_node = bus_sd->numa_node; + op->dev.dma_ops = bus->dev.dma_ops; if (dp->child) of_propagate_archdata(op); @@ -150,8 +153,8 @@ int of_bus_sbus_match(struct device_node *np) struct device_node *dp = np; while (dp) { - if (!strcmp(dp->name, "sbus") || - !strcmp(dp->name, "sbi")) + if (of_node_name_eq(dp, "sbus") || + of_node_name_eq(dp, "sbi")) return 1; /* Have a look at use_1to1_mapping(). We're trying @@ -159,7 +162,7 @@ int of_bus_sbus_match(struct device_node *np) * don't have some intervening real bus that provides * ranges based translations. */ - if (of_find_property(dp, "ranges", NULL) != NULL) + if (of_property_present(dp, "ranges")) break; dp = dp->parent; diff --git a/arch/sparc/kernel/of_device_common.h b/arch/sparc/kernel/of_device_common.h index cdfd23992841..3d66230c61b6 100644 --- a/arch/sparc/kernel/of_device_common.h +++ b/arch/sparc/kernel/of_device_common.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _OF_DEVICE_COMMON_H #define _OF_DEVICE_COMMON_H diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c index bc4d3f5d2e5d..a9448088e762 100644 --- a/arch/sparc/kernel/pci.c +++ b/arch/sparc/kernel/pci.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* pci.c: UltraSparc PCI controller support. * * Copyright (C) 1997, 1998, 1999 David S. Miller (davem@redhat.com) @@ -19,15 +20,17 @@ #include <linux/irq.h> #include <linux/init.h> #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/of_platform.h> +#include <linux/pgtable.h> +#include <linux/platform_device.h> -#include <asm/uaccess.h> -#include <asm/pgtable.h> +#include <linux/uaccess.h> #include <asm/irq.h> #include <asm/prom.h> #include <asm/apb.h> #include "pci_impl.h" +#include "kernel.h" /* List of all PCI controllers found in the system. */ struct pci_pbm_info *pci_pbm_root = NULL; @@ -184,8 +187,10 @@ static unsigned long pci_parse_of_flags(u32 addr0) if (addr0 & 0x02000000) { flags = IORESOURCE_MEM | PCI_BASE_ADDRESS_SPACE_MEMORY; - flags |= (addr0 >> 22) & PCI_BASE_ADDRESS_MEM_TYPE_64; flags |= (addr0 >> 28) & PCI_BASE_ADDRESS_MEM_TYPE_1M; + if (addr0 & 0x01000000) + flags |= IORESOURCE_MEM_64 + | PCI_BASE_ADDRESS_MEM_TYPE_64; if (addr0 & 0x40000000) flags |= IORESOURCE_PREFETCH | PCI_BASE_ADDRESS_MEM_PREFETCH; @@ -210,8 +215,8 @@ static void pci_parse_of_addrs(struct platform_device *op, if (!addrs) return; if (ofpci_verbose) - printk(" parse addresses (%d bytes) @ %p\n", - proplen, addrs); + pci_info(dev, " parse addresses (%d bytes) @ %p\n", + proplen, addrs); op_res = &op->resource[0]; for (; proplen >= 20; proplen -= 20, addrs += 5, op_res++) { struct resource *res; @@ -223,63 +228,67 @@ static void pci_parse_of_addrs(struct platform_device *op, continue; i = addrs[0] & 0xff; if (ofpci_verbose) - printk(" start: %llx, end: %llx, i: %x\n", - op_res->start, op_res->end, i); + pci_info(dev, " start: %llx, end: %llx, i: %x\n", + op_res->start, op_res->end, i); if (PCI_BASE_ADDRESS_0 <= i && i <= PCI_BASE_ADDRESS_5) { res = &dev->resource[(i - PCI_BASE_ADDRESS_0) >> 2]; } else if (i == dev->rom_base_reg) { res = &dev->resource[PCI_ROM_RESOURCE]; - flags |= IORESOURCE_READONLY | IORESOURCE_CACHEABLE - | IORESOURCE_SIZEALIGN; + flags |= IORESOURCE_READONLY | IORESOURCE_SIZEALIGN; } else { - printk(KERN_ERR "PCI: bad cfg reg num 0x%x\n", i); + pci_err(dev, "bad cfg reg num 0x%x\n", i); continue; } res->start = op_res->start; res->end = op_res->end; res->flags = flags; res->name = pci_name(dev); + + pci_info(dev, "reg 0x%x: %pR\n", i, res); } } +static void pci_init_dev_archdata(struct dev_archdata *sd, void *iommu, + void *stc, void *host_controller, + struct platform_device *op, + int numa_node) +{ + sd->iommu = iommu; + sd->stc = stc; + sd->host_controller = host_controller; + sd->op = op; + sd->numa_node = numa_node; +} + static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, struct device_node *node, struct pci_bus *bus, int devfn) { struct dev_archdata *sd; - struct pci_slot *slot; struct platform_device *op; struct pci_dev *dev; - const char *type; u32 class; dev = pci_alloc_dev(bus); if (!dev) return NULL; + op = of_find_device_by_node(node); sd = &dev->dev.archdata; - sd->iommu = pbm->iommu; - sd->stc = &pbm->stc; - sd->host_controller = pbm; - sd->op = op = of_find_device_by_node(node); - sd->numa_node = pbm->numa_node; - + pci_init_dev_archdata(sd, pbm->iommu, &pbm->stc, pbm, op, + pbm->numa_node); sd = &op->dev.archdata; sd->iommu = pbm->iommu; sd->stc = &pbm->stc; sd->numa_node = pbm->numa_node; - if (!strcmp(node->name, "ebus")) + if (of_node_name_eq(node, "ebus")) of_propagate_archdata(op); - type = of_get_property(node, "device_type", NULL); - if (type == NULL) - type = ""; - if (ofpci_verbose) - printk(" create device, devfn: %x, type: %s\n", - devfn, type); + pci_info(bus," create device, devfn: %x, type: %s\n", + devfn, of_node_get_device_type(node)); dev->sysdata = node; dev->dev.parent = bus->bridge; @@ -289,10 +298,7 @@ static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, dev->multifunction = 0; /* maybe a lie? */ set_pcie_port_type(dev); - list_for_each_entry(slot, &dev->bus->slots, list) - if (PCI_SLOT(dev->devfn) == slot->number) - dev->slot = slot; - + pci_dev_assign_slot(dev); dev->vendor = of_getintprop_default(node, "vendor-id", 0xffff); dev->device = of_getintprop_default(node, "device-id", 0xffff); dev->subsystem_vendor = @@ -305,7 +311,7 @@ static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, /* We can't actually use the firmware value, we have * to read what is in the register right now. One * reason is that in the case of IDE interfaces the - * firmware can sample the value before the the IDE + * firmware can sample the value before the IDE * interface is programmed into native mode. */ pci_read_config_dword(dev, PCI_CLASS_REVISION, &class); @@ -315,10 +321,6 @@ static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, dev_set_name(&dev->dev, "%04x:%02x:%02x.%d", pci_domain_nr(bus), dev->bus->number, PCI_SLOT(devfn), PCI_FUNC(devfn)); - if (ofpci_verbose) - printk(" class: 0x%x device name: %s\n", - dev->class, pci_name(dev)); - /* I have seen IDE devices which will not respond to * the bmdma simplex check reads if bus mastering is * disabled. @@ -330,11 +332,11 @@ static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, dev->error_state = pci_channel_io_normal; dev->dma_mask = 0xffffffff; - if (!strcmp(node->name, "pci")) { + if (of_node_name_eq(node, "pci")) { /* a PCI-PCI bridge */ dev->hdr_type = PCI_HEADER_TYPE_BRIDGE; dev->rom_base_reg = PCI_ROM_ADDRESS1; - } else if (!strcmp(type, "cardbus")) { + } else if (of_node_is_type(node, "cardbus")) { dev->hdr_type = PCI_HEADER_TYPE_CARDBUS; } else { dev->hdr_type = PCI_HEADER_TYPE_NORMAL; @@ -345,10 +347,13 @@ static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, dev->irq = PCI_IRQ_NONE; } + pci_info(dev, "[%04x:%04x] type %02x class %#08x\n", + dev->vendor, dev->device, dev->hdr_type, dev->class); + pci_parse_of_addrs(sd->op, node, dev); if (ofpci_verbose) - printk(" adding to system ...\n"); + pci_info(dev, " adding to system ...\n"); pci_device_add(dev, bus); @@ -392,15 +397,15 @@ static void apb_fake_ranges(struct pci_dev *dev, res->flags = IORESOURCE_IO; region.start = (first << 21); region.end = (last << 21) + ((1 << 21) - 1); - pcibios_bus_to_resource(dev, res, ®ion); + pcibios_bus_to_resource(dev->bus, res, ®ion); pci_read_config_byte(dev, APB_MEM_ADDRESS_MAP, &map); apb_calc_first_last(map, &first, &last); res = bus->resource[1]; res->flags = IORESOURCE_MEM; - region.start = (first << 21); - region.end = (last << 21) + ((1 << 21) - 1); - pcibios_bus_to_resource(dev, res, ®ion); + region.start = (first << 29); + region.end = (last << 29) + ((1 << 29) - 1); + pcibios_bus_to_resource(dev->bus, res, ®ion); } static void pci_of_scan_bus(struct pci_pbm_info *pbm, @@ -422,15 +427,20 @@ static void of_scan_pci_bridge(struct pci_pbm_info *pbm, u64 size; if (ofpci_verbose) - printk("of_scan_pci_bridge(%s)\n", node->full_name); + pci_info(dev, "of_scan_pci_bridge(%pOF)\n", node); /* parse bus-range property */ busrange = of_get_property(node, "bus-range", &len); if (busrange == NULL || len != 8) { - printk(KERN_DEBUG "Can't get bus-range for PCI-PCI bridge %s\n", - node->full_name); + pci_info(dev, "Can't get bus-range for PCI-PCI bridge %pOF\n", + node); return; } + + if (ofpci_verbose) + pci_info(dev, " Bridge bus range [%u --> %u]\n", + busrange[0], busrange[1]); + ranges = of_get_property(node, "ranges", &len); simba = 0; if (ranges == NULL) { @@ -441,8 +451,8 @@ static void of_scan_pci_bridge(struct pci_pbm_info *pbm, bus = pci_add_new_bus(dev->bus, dev, busrange[0]); if (!bus) { - printk(KERN_ERR "Failed to create pci bus for %s\n", - node->full_name); + pci_err(dev, "Failed to create pci bus for %pOF\n", + node); return; } @@ -450,6 +460,10 @@ static void of_scan_pci_bridge(struct pci_pbm_info *pbm, pci_bus_insert_busn_res(bus, busrange[0], busrange[1]); bus->bridge_ctl = 0; + if (ofpci_verbose) + pci_info(dev, " Bridge ranges[%p] simba[%d]\n", + ranges, simba); + /* parse ranges property, or cook one up by hand for Simba */ /* PCI #address-cells == 3 and #size-cells == 2 always */ res = &dev->resource[PCI_BRIDGE_RESOURCES]; @@ -467,21 +481,40 @@ static void of_scan_pci_bridge(struct pci_pbm_info *pbm, } i = 1; for (; len >= 32; len -= 32, ranges += 8) { + u64 start; + + if (ofpci_verbose) + pci_info(dev, " RAW Range[%08x:%08x:%08x:%08x:%08x:%08x:" + "%08x:%08x]\n", + ranges[0], ranges[1], ranges[2], ranges[3], + ranges[4], ranges[5], ranges[6], ranges[7]); + flags = pci_parse_of_flags(ranges[0]); size = GET_64BIT(ranges, 6); if (flags == 0 || size == 0) continue; + + /* On PCI-Express systems, PCI bridges that have no devices downstream + * have a bogus size value where the first 32-bit cell is 0xffffffff. + * This results in a bogus range where start + size overflows. + * + * Just skip these otherwise the kernel will complain when the resource + * tries to be claimed. + */ + if (size >> 32 == 0xffffffff) + continue; + if (flags & IORESOURCE_IO) { res = bus->resource[0]; if (res->flags) { - printk(KERN_ERR "PCI: ignoring extra I/O range" - " for bridge %s\n", node->full_name); + pci_err(dev, "ignoring extra I/O range" + " for bridge %pOF\n", node); continue; } } else { if (i >= PCI_NUM_RESOURCES - PCI_BRIDGE_RESOURCES) { - printk(KERN_ERR "PCI: too many memory ranges" - " for bridge %s\n", node->full_name); + pci_err(dev, "too many memory ranges" + " for bridge %pOF\n", node); continue; } res = bus->resource[i]; @@ -489,15 +522,20 @@ static void of_scan_pci_bridge(struct pci_pbm_info *pbm, } res->flags = flags; - region.start = GET_64BIT(ranges, 1); + region.start = start = GET_64BIT(ranges, 1); region.end = region.start + size - 1; - pcibios_bus_to_resource(dev, res, ®ion); + + if (ofpci_verbose) + pci_info(dev, " Using flags[%08x] start[%016llx] size[%016llx]\n", + flags, start, size); + + pcibios_bus_to_resource(dev->bus, res, ®ion); } after_ranges: sprintf(bus->name, "PCI Bus %04x:%02x", pci_domain_nr(bus), bus->number); if (ofpci_verbose) - printk(" bus name: %s\n", bus->name); + pci_info(dev, " bus name: %s\n", bus->name); pci_of_scan_bus(pbm, node, bus); } @@ -512,14 +550,13 @@ static void pci_of_scan_bus(struct pci_pbm_info *pbm, struct pci_dev *dev; if (ofpci_verbose) - printk("PCI: scan_bus[%s] bus no %d\n", - node->full_name, bus->number); + pci_info(bus, "scan_bus[%pOF] bus no %d\n", + node, bus->number); - child = NULL; prev_devfn = -1; - while ((child = of_get_next_child(node, child)) != NULL) { + for_each_child_of_node(node, child) { if (ofpci_verbose) - printk(" * %s\n", child->full_name); + pci_info(bus, " * %pOF\n", child); reg = of_get_property(child, "reg", ®len); if (reg == NULL || reglen < 20) continue; @@ -540,11 +577,9 @@ static void pci_of_scan_bus(struct pci_pbm_info *pbm, if (!dev) continue; if (ofpci_verbose) - printk("PCI: dev header type: %x\n", - dev->hdr_type); + pci_info(dev, "dev header type: %x\n", dev->hdr_type); - if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE || - dev->hdr_type == PCI_HEADER_TYPE_CARDBUS) + if (pci_is_bridge(dev)) of_scan_pci_bridge(pbm, child, dev); } } @@ -558,7 +593,7 @@ show_pciobppath_attr(struct device * dev, struct device_attribute * attr, char * pdev = to_pci_dev(dev); dp = pdev->dev.of_node; - return snprintf (buf, PAGE_SIZE, "%s\n", dp->full_name); + return scnprintf(buf, PAGE_SIZE, "%pOF\n", dp); } static DEVICE_ATTR(obppath, S_IRUSR | S_IRGRP | S_IROTH, show_pciobppath_attr, NULL); @@ -584,6 +619,72 @@ static void pci_bus_register_of_sysfs(struct pci_bus *bus) pci_bus_register_of_sysfs(child_bus); } +static void pci_claim_legacy_resources(struct pci_dev *dev) +{ + struct pci_bus_region region; + struct resource *p, *root, *conflict; + + if ((dev->class >> 8) != PCI_CLASS_DISPLAY_VGA) + return; + + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (!p) + return; + + p->name = "Video RAM area"; + p->flags = IORESOURCE_MEM | IORESOURCE_BUSY; + + region.start = 0xa0000UL; + region.end = region.start + 0x1ffffUL; + pcibios_bus_to_resource(dev->bus, p, ®ion); + + root = pci_find_parent_resource(dev, p); + if (!root) { + pci_info(dev, "can't claim VGA legacy %pR: no compatible bridge window\n", p); + goto err; + } + + conflict = request_resource_conflict(root, p); + if (conflict) { + pci_info(dev, "can't claim VGA legacy %pR: address conflict with %s %pR\n", + p, conflict->name, conflict); + goto err; + } + + pci_info(dev, "VGA legacy framebuffer %pR\n", p); + return; + +err: + kfree(p); +} + +static void pci_claim_bus_resources(struct pci_bus *bus) +{ + struct pci_bus *child_bus; + struct pci_dev *dev; + + list_for_each_entry(dev, &bus->devices, bus_list) { + struct resource *r; + int i; + + pci_dev_for_each_resource(dev, r, i) { + if (r->parent || !r->start || !r->flags) + continue; + + if (ofpci_verbose) + pci_info(dev, "Claiming Resource %d: %pR\n", + i, r); + + pci_claim_resource(dev, i); + } + + pci_claim_legacy_resources(dev); + } + + list_for_each_entry(child_bus, &bus->children, node) + pci_claim_bus_resources(child_bus); +} + struct pci_bus *pci_scan_one_pbm(struct pci_pbm_info *pbm, struct device *parent) { @@ -591,12 +692,15 @@ struct pci_bus *pci_scan_one_pbm(struct pci_pbm_info *pbm, struct device_node *node = pbm->op->dev.of_node; struct pci_bus *bus; - printk("PCI: Scanning PBM %s\n", node->full_name); + printk("PCI: Scanning PBM %pOF\n", node); pci_add_resource_offset(&resources, &pbm->io_space, - pbm->io_space.start); + pbm->io_offset); pci_add_resource_offset(&resources, &pbm->mem_space, - pbm->mem_space.start); + pbm->mem_offset); + if (pbm->mem64_space.flags) + pci_add_resource_offset(&resources, &pbm->mem64_space, + pbm->mem64_offset); pbm->busn.start = pbm->pci_first_busno; pbm->busn.end = pbm->pci_last_busno; pbm->busn.flags = IORESOURCE_BUS; @@ -604,210 +708,30 @@ struct pci_bus *pci_scan_one_pbm(struct pci_pbm_info *pbm, bus = pci_create_root_bus(parent, pbm->pci_first_busno, pbm->pci_ops, pbm, &resources); if (!bus) { - printk(KERN_ERR "Failed to create bus for %s\n", - node->full_name); + printk(KERN_ERR "Failed to create bus for %pOF\n", node); pci_free_resource_list(&resources); return NULL; } pci_of_scan_bus(pbm, node, bus); - pci_bus_add_devices(bus); pci_bus_register_of_sysfs(bus); - return bus; -} - -void pcibios_fixup_bus(struct pci_bus *pbus) -{ -} - -resource_size_t pcibios_align_resource(void *data, const struct resource *res, - resource_size_t size, resource_size_t align) -{ - return res->start; -} - -int pcibios_enable_device(struct pci_dev *dev, int mask) -{ - u16 cmd, oldcmd; - int i; + pci_claim_bus_resources(bus); - pci_read_config_word(dev, PCI_COMMAND, &cmd); - oldcmd = cmd; - - for (i = 0; i < PCI_NUM_RESOURCES; i++) { - struct resource *res = &dev->resource[i]; - - /* Only set up the requested stuff */ - if (!(mask & (1<<i))) - continue; - - if (res->flags & IORESOURCE_IO) - cmd |= PCI_COMMAND_IO; - if (res->flags & IORESOURCE_MEM) - cmd |= PCI_COMMAND_MEMORY; - } - - if (cmd != oldcmd) { - printk(KERN_DEBUG "PCI: Enabling device: (%s), cmd %x\n", - pci_name(dev), cmd); - /* Enable the appropriate bits in the PCI command register. */ - pci_write_config_word(dev, PCI_COMMAND, cmd); - } - return 0; + pci_bus_add_devices(bus); + return bus; } /* Platform support for /proc/bus/pci/X/Y mmap()s. */ - -/* If the user uses a host-bridge as the PCI device, he may use - * this to perform a raw mmap() of the I/O or MEM space behind - * that controller. - * - * This can be useful for execution of x86 PCI bios initialization code - * on a PCI card, like the xfree86 int10 stuff does. - */ -static int __pci_mmap_make_offset_bus(struct pci_dev *pdev, struct vm_area_struct *vma, - enum pci_mmap_state mmap_state) +int pci_iobar_pfn(struct pci_dev *pdev, int bar, struct vm_area_struct *vma) { struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; - unsigned long space_size, user_offset, user_size; - - if (mmap_state == pci_mmap_io) { - space_size = resource_size(&pbm->io_space); - } else { - space_size = resource_size(&pbm->mem_space); - } - - /* Make sure the request is in range. */ - user_offset = vma->vm_pgoff << PAGE_SHIFT; - user_size = vma->vm_end - vma->vm_start; + resource_size_t ioaddr = pci_resource_start(pdev, bar); - if (user_offset >= space_size || - (user_offset + user_size) > space_size) + if (!pbm) return -EINVAL; - if (mmap_state == pci_mmap_io) { - vma->vm_pgoff = (pbm->io_space.start + - user_offset) >> PAGE_SHIFT; - } else { - vma->vm_pgoff = (pbm->mem_space.start + - user_offset) >> PAGE_SHIFT; - } - - return 0; -} - -/* Adjust vm_pgoff of VMA such that it is the physical page offset - * corresponding to the 32-bit pci bus offset for DEV requested by the user. - * - * Basically, the user finds the base address for his device which he wishes - * to mmap. They read the 32-bit value from the config space base register, - * add whatever PAGE_SIZE multiple offset they wish, and feed this into the - * offset parameter of mmap on /proc/bus/pci/XXX for that device. - * - * Returns negative error code on failure, zero on success. - */ -static int __pci_mmap_make_offset(struct pci_dev *pdev, - struct vm_area_struct *vma, - enum pci_mmap_state mmap_state) -{ - unsigned long user_paddr, user_size; - int i, err; - - /* First compute the physical address in vma->vm_pgoff, - * making sure the user offset is within range in the - * appropriate PCI space. - */ - err = __pci_mmap_make_offset_bus(pdev, vma, mmap_state); - if (err) - return err; - - /* If this is a mapping on a host bridge, any address - * is OK. - */ - if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_HOST) - return err; - - /* Otherwise make sure it's in the range for one of the - * device's resources. - */ - user_paddr = vma->vm_pgoff << PAGE_SHIFT; - user_size = vma->vm_end - vma->vm_start; - - for (i = 0; i <= PCI_ROM_RESOURCE; i++) { - struct resource *rp = &pdev->resource[i]; - resource_size_t aligned_end; - - /* Active? */ - if (!rp->flags) - continue; - - /* Same type? */ - if (i == PCI_ROM_RESOURCE) { - if (mmap_state != pci_mmap_mem) - continue; - } else { - if ((mmap_state == pci_mmap_io && - (rp->flags & IORESOURCE_IO) == 0) || - (mmap_state == pci_mmap_mem && - (rp->flags & IORESOURCE_MEM) == 0)) - continue; - } - - /* Align the resource end to the next page address. - * PAGE_SIZE intentionally added instead of (PAGE_SIZE - 1), - * because actually we need the address of the next byte - * after rp->end. - */ - aligned_end = (rp->end + PAGE_SIZE) & PAGE_MASK; - - if ((rp->start <= user_paddr) && - (user_paddr + user_size) <= aligned_end) - break; - } - - if (i > PCI_ROM_RESOURCE) - return -EINVAL; - - return 0; -} - -/* Set vm_page_prot of VMA, as appropriate for this architecture, for a pci - * device mapping. - */ -static void __pci_mmap_set_pgprot(struct pci_dev *dev, struct vm_area_struct *vma, - enum pci_mmap_state mmap_state) -{ - /* Our io_remap_pfn_range takes care of this, do nothing. */ -} - -/* Perform the actual remap of the pages for a PCI device mapping, as appropriate - * for this architecture. The region in the process to map is described by vm_start - * and vm_end members of VMA, the base physical address is found in vm_pgoff. - * The pci device structure is provided so that architectures may make mapping - * decisions on a per-device or per-bus basis. - * - * Returns a negative error code on failure, zero on success. - */ -int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, - enum pci_mmap_state mmap_state, - int write_combine) -{ - int ret; - - ret = __pci_mmap_make_offset(dev, vma, mmap_state); - if (ret < 0) - return ret; - - __pci_mmap_set_pgprot(dev, vma, mmap_state); - - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - ret = io_remap_pfn_range(vma, vma->vm_start, - vma->vm_pgoff, - vma->vm_end - vma->vm_start, - vma->vm_page_prot); - if (ret) - return ret; + vma->vm_pgoff += (ioaddr + pbm->io_space.start) >> PAGE_SHIFT; return 0; } @@ -854,7 +778,7 @@ int arch_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc) void arch_teardown_msi_irq(unsigned int irq) { struct msi_desc *entry = irq_get_msi_desc(irq); - struct pci_dev *pdev = entry->dev; + struct pci_dev *pdev = msi_desc_to_pci_dev(entry); struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; if (pbm->teardown_msi_irq) @@ -862,67 +786,53 @@ void arch_teardown_msi_irq(unsigned int irq) } #endif /* !(CONFIG_PCI_MSI) */ -static void ali_sound_dma_hack(struct pci_dev *pdev, int set_bit) +/* ALI sound chips generate 31-bits of DMA, a special register + * determines what bit 31 is emitted as. + */ +int ali_sound_dma_hack(struct device *dev, u64 device_mask) { + struct iommu *iommu = dev->archdata.iommu; struct pci_dev *ali_isa_bridge; u8 val; - /* ALI sound chips generate 31-bits of DMA, a special register - * determines what bit 31 is emitted as. - */ + if (!dev_is_pci(dev)) + return 0; + + if (to_pci_dev(dev)->vendor != PCI_VENDOR_ID_AL || + to_pci_dev(dev)->device != PCI_DEVICE_ID_AL_M5451 || + device_mask != 0x7fffffff) + return 0; + ali_isa_bridge = pci_get_device(PCI_VENDOR_ID_AL, PCI_DEVICE_ID_AL_M1533, NULL); pci_read_config_byte(ali_isa_bridge, 0x7e, &val); - if (set_bit) + if (iommu->dma_addr_mask & 0x80000000) val |= 0x01; else val &= ~0x01; pci_write_config_byte(ali_isa_bridge, 0x7e, val); pci_dev_put(ali_isa_bridge); -} - -int pci64_dma_supported(struct pci_dev *pdev, u64 device_mask) -{ - u64 dma_addr_mask; - - if (pdev == NULL) { - dma_addr_mask = 0xffffffff; - } else { - struct iommu *iommu = pdev->dev.archdata.iommu; - - dma_addr_mask = iommu->dma_addr_mask; - - if (pdev->vendor == PCI_VENDOR_ID_AL && - pdev->device == PCI_DEVICE_ID_AL_M5451 && - device_mask == 0x7fffffff) { - ali_sound_dma_hack(pdev, - (dma_addr_mask & 0x80000000) != 0); - return 1; - } - } - - if (device_mask >= (1UL << 32UL)) - return 0; - - return (device_mask & dma_addr_mask) == dma_addr_mask; + return 1; } void pci_resource_to_user(const struct pci_dev *pdev, int bar, const struct resource *rp, resource_size_t *start, resource_size_t *end) { - struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; - unsigned long offset; - - if (rp->flags & IORESOURCE_IO) - offset = pbm->io_space.start; - else - offset = pbm->mem_space.start; + struct pci_bus_region region; - *start = rp->start - offset; - *end = rp->end - offset; + /* + * "User" addresses are shown in /sys/devices/pci.../.../resource + * and /proc/bus/pci/devices and used as mmap offsets for + * /proc/bus/pci/BB/DD.F files (see proc_bus_pci_mmap()). + * + * On sparc, these are PCI bus addresses, i.e., raw BAR values. + */ + pcibios_resource_to_bus(pdev->bus, ®ion, (struct resource *) rp); + *start = region.start; + *end = region.end; } void pcibios_set_master(struct pci_dev *dev) @@ -930,6 +840,27 @@ void pcibios_set_master(struct pci_dev *dev) /* No special bus mastering setup handling */ } +#ifdef CONFIG_PCI_IOV +int pcibios_device_add(struct pci_dev *dev) +{ + struct pci_dev *pdev; + + /* Add sriov arch specific initialization here. + * Copy dev_archdata from PF to VF + */ + if (dev->is_virtfn) { + struct dev_archdata *psd; + + pdev = dev->physfn; + psd = &pdev->dev.archdata; + pci_init_dev_archdata(&dev->dev.archdata, psd->iommu, + psd->stc, psd->host_controller, NULL, + psd->numa_node); + } + return 0; +} +#endif /* CONFIG_PCI_IOV */ + static int __init pcibios_init(void) { pci_dfl_cache_line_size = 64 >> 2; @@ -938,11 +869,43 @@ static int __init pcibios_init(void) subsys_initcall(pcibios_init); #ifdef CONFIG_SYSFS + +#define SLOT_NAME_SIZE 11 /* Max decimal digits + null in u32 */ + +static void pcie_bus_slot_names(struct pci_bus *pbus) +{ + struct pci_dev *pdev; + struct pci_bus *bus; + + list_for_each_entry(pdev, &pbus->devices, bus_list) { + char name[SLOT_NAME_SIZE]; + struct pci_slot *pci_slot; + const u32 *slot_num; + int len; + + slot_num = of_get_property(pdev->dev.of_node, + "physical-slot#", &len); + + if (slot_num == NULL || len != 4) + continue; + + snprintf(name, sizeof(name), "%u", slot_num[0]); + pci_slot = pci_create_slot(pbus, slot_num[0], name, NULL); + + if (IS_ERR(pci_slot)) + pr_err("PCI: pci_create_slot returned %ld.\n", + PTR_ERR(pci_slot)); + } + + list_for_each_entry(bus, &pbus->children, node) + pcie_bus_slot_names(bus); +} + static void pci_bus_slot_names(struct device_node *node, struct pci_bus *bus) { const struct pci_slot_names { u32 slot_mask; - char names[0]; + char names[]; } *prop; const char *sp; int len, i; @@ -956,8 +919,8 @@ static void pci_bus_slot_names(struct device_node *node, struct pci_bus *bus) sp = prop->names; if (ofpci_verbose) - printk("PCI: Making slots for [%s] mask[0x%02x]\n", - node->full_name, mask); + pci_info(bus, "Making slots for [%pOF] mask[0x%02x]\n", + node, mask); i = 0; while (mask) { @@ -970,12 +933,12 @@ static void pci_bus_slot_names(struct device_node *node, struct pci_bus *bus) } if (ofpci_verbose) - printk("PCI: Making slot [%s]\n", sp); + pci_info(bus, "Making slot [%s]\n", sp); pci_slot = pci_create_slot(bus, i, sp, NULL); if (IS_ERR(pci_slot)) - printk(KERN_ERR "PCI: pci_create_slot returned %ld\n", - PTR_ERR(pci_slot)); + pci_err(bus, "pci_create_slot returned %ld\n", + PTR_ERR(pci_slot)); sp += strlen(sp) + 1; mask &= ~this_bit; @@ -989,22 +952,32 @@ static int __init of_pci_slot_init(void) while ((pbus = pci_find_next_bus(pbus)) != NULL) { struct device_node *node; + struct pci_dev *pdev; + + pdev = list_first_entry(&pbus->devices, struct pci_dev, + bus_list); - if (pbus->self) { - /* PCI->PCI bridge */ - node = pbus->self->dev.of_node; + if (pdev && pci_is_pcie(pdev)) { + pcie_bus_slot_names(pbus); } else { - struct pci_pbm_info *pbm = pbus->sysdata; - /* Host PCI controller */ - node = pbm->op->dev.of_node; - } + if (pbus->self) { - pci_bus_slot_names(node, pbus); + /* PCI->PCI bridge */ + node = pbus->self->dev.of_node; + + } else { + struct pci_pbm_info *pbm = pbus->sysdata; + + /* Host PCI controller */ + node = pbm->op->dev.of_node; + } + + pci_bus_slot_names(node, pbus); + } } return 0; } - -module_init(of_pci_slot_init); +device_initcall(of_pci_slot_init); #endif diff --git a/arch/sparc/kernel/pci_common.c b/arch/sparc/kernel/pci_common.c index a6895987fb70..2576f4f31309 100644 --- a/arch/sparc/kernel/pci_common.c +++ b/arch/sparc/kernel/pci_common.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* pci_common.c: PCI controller common support. * * Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net) @@ -5,10 +6,10 @@ #include <linux/string.h> #include <linux/slab.h> -#include <linux/init.h> #include <linux/pci.h> #include <linux/device.h> -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/platform_device.h> #include <asm/prom.h> #include <asm/oplib.h> @@ -329,43 +330,6 @@ void pci_get_pbm_props(struct pci_pbm_info *pbm) } } -static void pci_register_legacy_regions(struct resource *io_res, - struct resource *mem_res) -{ - struct resource *p; - - /* VGA Video RAM. */ - p = kzalloc(sizeof(*p), GFP_KERNEL); - if (!p) - return; - - p->name = "Video RAM area"; - p->start = mem_res->start + 0xa0000UL; - p->end = p->start + 0x1ffffUL; - p->flags = IORESOURCE_BUSY; - request_resource(mem_res, p); - - p = kzalloc(sizeof(*p), GFP_KERNEL); - if (!p) - return; - - p->name = "System ROM"; - p->start = mem_res->start + 0xf0000UL; - p->end = p->start + 0xffffUL; - p->flags = IORESOURCE_BUSY; - request_resource(mem_res, p); - - p = kzalloc(sizeof(*p), GFP_KERNEL); - if (!p) - return; - - p->name = "Video ROM"; - p->start = mem_res->start + 0xc0000UL; - p->end = p->start + 0x7fffUL; - p->flags = IORESOURCE_BUSY; - request_resource(mem_res, p); -} - static void pci_register_iommu_region(struct pci_pbm_info *pbm) { const u32 *vdma = of_get_property(pbm->op->dev.of_node, "virtual-dma", @@ -397,6 +361,8 @@ void pci_determine_mem_io_space(struct pci_pbm_info *pbm) int i, saw_mem, saw_io; int num_pbm_ranges; + /* Corresponds to generic devm_of_pci_get_host_bridge_resources() */ + saw_mem = saw_io = 0; pbm_ranges = of_get_property(pbm->op->dev.of_node, "ranges", &i); if (!pbm_ranges) { @@ -407,16 +373,20 @@ void pci_determine_mem_io_space(struct pci_pbm_info *pbm) } num_pbm_ranges = i / sizeof(*pbm_ranges); + memset(&pbm->mem64_space, 0, sizeof(struct resource)); for (i = 0; i < num_pbm_ranges; i++) { const struct linux_prom_pci_ranges *pr = &pbm_ranges[i]; - unsigned long a, size; + unsigned long a, size, region_a; u32 parent_phys_hi, parent_phys_lo; + u32 child_phys_mid, child_phys_lo; u32 size_hi, size_lo; int type; parent_phys_hi = pr->parent_phys_hi; parent_phys_lo = pr->parent_phys_lo; + child_phys_mid = pr->child_phys_mid; + child_phys_lo = pr->child_phys_lo; if (tlb_type == hypervisor) parent_phys_hi &= 0x0fffffff; @@ -426,6 +396,8 @@ void pci_determine_mem_io_space(struct pci_pbm_info *pbm) type = (pr->child_phys_hi >> 24) & 0x3; a = (((unsigned long)parent_phys_hi << 32UL) | ((unsigned long)parent_phys_lo << 0UL)); + region_a = (((unsigned long)child_phys_mid << 32UL) | + ((unsigned long)child_phys_lo << 0UL)); size = (((unsigned long)size_hi << 32UL) | ((unsigned long)size_lo << 0UL)); @@ -440,6 +412,7 @@ void pci_determine_mem_io_space(struct pci_pbm_info *pbm) pbm->io_space.start = a; pbm->io_space.end = a + size - 1UL; pbm->io_space.flags = IORESOURCE_IO; + pbm->io_offset = a - region_a; saw_io = 1; break; @@ -448,11 +421,18 @@ void pci_determine_mem_io_space(struct pci_pbm_info *pbm) pbm->mem_space.start = a; pbm->mem_space.end = a + size - 1UL; pbm->mem_space.flags = IORESOURCE_MEM; + pbm->mem_offset = a - region_a; saw_mem = 1; break; case 3: - /* XXX 64-bit MEM handling XXX */ + /* 64-bit MEM handling */ + pbm->mem64_space.start = a; + pbm->mem64_space.end = a + size - 1UL; + pbm->mem64_space.flags = IORESOURCE_MEM; + pbm->mem64_offset = a - region_a; + saw_mem = 1; + break; default: break; @@ -466,18 +446,31 @@ void pci_determine_mem_io_space(struct pci_pbm_info *pbm) prom_halt(); } - printk("%s: PCI IO[%llx] MEM[%llx]\n", - pbm->name, - pbm->io_space.start, - pbm->mem_space.start); + if (pbm->io_space.flags) + printk("%s: PCI IO %pR offset %llx\n", + pbm->name, &pbm->io_space, pbm->io_offset); + if (pbm->mem_space.flags) + printk("%s: PCI MEM %pR offset %llx\n", + pbm->name, &pbm->mem_space, pbm->mem_offset); + if (pbm->mem64_space.flags && pbm->mem_space.flags) { + if (pbm->mem64_space.start <= pbm->mem_space.end) + pbm->mem64_space.start = pbm->mem_space.end + 1; + if (pbm->mem64_space.start > pbm->mem64_space.end) + pbm->mem64_space.flags = 0; + } + + if (pbm->mem64_space.flags) + printk("%s: PCI MEM64 %pR offset %llx\n", + pbm->name, &pbm->mem64_space, pbm->mem64_offset); pbm->io_space.name = pbm->mem_space.name = pbm->name; + pbm->mem64_space.name = pbm->name; request_resource(&ioport_resource, &pbm->io_space); request_resource(&iomem_resource, &pbm->mem_space); + if (pbm->mem64_space.flags) + request_resource(&iomem_resource, &pbm->mem64_space); - pci_register_legacy_regions(&pbm->io_space, - &pbm->mem_space); pci_register_iommu_region(pbm); } @@ -497,8 +490,8 @@ void pci_scan_for_target_abort(struct pci_pbm_info *pbm, PCI_STATUS_REC_TARGET_ABORT)); if (error_bits) { pci_write_config_word(pdev, PCI_STATUS, error_bits); - printk("%s: Device %s saw Target Abort [%016x]\n", - pbm->name, pci_name(pdev), status); + pci_info(pdev, "%s: Device saw Target Abort [%016x]\n", + pbm->name, status); } } @@ -520,8 +513,8 @@ void pci_scan_for_master_abort(struct pci_pbm_info *pbm, (status & (PCI_STATUS_REC_MASTER_ABORT)); if (error_bits) { pci_write_config_word(pdev, PCI_STATUS, error_bits); - printk("%s: Device %s received Master Abort [%016x]\n", - pbm->name, pci_name(pdev), status); + pci_info(pdev, "%s: Device received Master Abort " + "[%016x]\n", pbm->name, status); } } @@ -544,8 +537,8 @@ void pci_scan_for_parity_error(struct pci_pbm_info *pbm, PCI_STATUS_DETECTED_PARITY)); if (error_bits) { pci_write_config_word(pdev, PCI_STATUS, error_bits); - printk("%s: Device %s saw Parity Error [%016x]\n", - pbm->name, pci_name(pdev), status); + pci_info(pdev, "%s: Device saw Parity Error [%016x]\n", + pbm->name, status); } } diff --git a/arch/sparc/kernel/pci_fire.c b/arch/sparc/kernel/pci_fire.c index e60fc6a67e9b..0b91bde80fdc 100644 --- a/arch/sparc/kernel/pci_fire.c +++ b/arch/sparc/kernel/pci_fire.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* pci_fire.c: Sun4u platform PCI-E controller support. * * Copyright (C) 2007 David S. Miller (davem@davemloft.net) @@ -9,7 +10,9 @@ #include <linux/msi.h> #include <linux/export.h> #include <linux/irq.h> -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/numa.h> #include <asm/prom.h> #include <asm/irq.h> @@ -415,7 +418,7 @@ static int pci_fire_pbm_init(struct pci_pbm_info *pbm, struct device_node *dp = op->dev.of_node; int err; - pbm->numa_node = -1; + pbm->numa_node = NUMA_NO_NODE; pbm->pci_ops = &sun4u_pci_ops; pbm->config_space_reg_bits = 12; @@ -508,7 +511,6 @@ static const struct of_device_id fire_match[] = { static struct platform_driver fire_driver = { .driver = { .name = DRIVER_NAME, - .owner = THIS_MODULE, .of_match_table = fire_match, }, .probe = fire_probe, diff --git a/arch/sparc/kernel/pci_impl.h b/arch/sparc/kernel/pci_impl.h index 5f688531f48c..83718876f1d4 100644 --- a/arch/sparc/kernel/pci_impl.h +++ b/arch/sparc/kernel/pci_impl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* pci_impl.h: Helper definitions for PCI controller support. * * Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net) @@ -10,7 +11,6 @@ #include <linux/spinlock.h> #include <linux/pci.h> #include <linux/msi.h> -#include <linux/of_device.h> #include <asm/io.h> #include <asm/prom.h> #include <asm/iommu.h> @@ -19,9 +19,9 @@ * each with one (Sabre) or two (PSYCHO/SCHIZO) PCI bus modules * underneath. Each PCI bus module uses an IOMMU (shared by both * PBMs of a controller, or per-PBM), and if a streaming buffer - * is present, each PCI bus module has it's own. (ie. the IOMMU + * is present, each PCI bus module has its own. (ie. the IOMMU * might be shared between PBMs, the STC is never shared) - * Furthermore, each PCI bus module controls it's own autonomous + * Furthermore, each PCI bus module controls its own autonomous * PCI bus. */ @@ -48,8 +48,8 @@ struct sparc64_msiq_ops { unsigned long devino); }; -extern void sparc64_pbm_msi_init(struct pci_pbm_info *pbm, - const struct sparc64_msiq_ops *ops); +void sparc64_pbm_msi_init(struct pci_pbm_info *pbm, + const struct sparc64_msiq_ops *ops); struct sparc64_msiq_cookie { struct pci_pbm_info *pbm; @@ -97,7 +97,12 @@ struct pci_pbm_info { /* PBM I/O and Memory space resources. */ struct resource io_space; struct resource mem_space; + struct resource mem64_space; struct resource busn; + /* offset */ + resource_size_t io_offset; + resource_size_t mem_offset; + resource_size_t mem64_offset; /* Base of PCI Config space, can be per-PBM or shared. */ unsigned long config_space; @@ -158,23 +163,23 @@ extern struct pci_pbm_info *pci_pbm_root; extern int pci_num_pbms; /* PCI bus scanning and fixup support. */ -extern void pci_get_pbm_props(struct pci_pbm_info *pbm); -extern struct pci_bus *pci_scan_one_pbm(struct pci_pbm_info *pbm, - struct device *parent); -extern void pci_determine_mem_io_space(struct pci_pbm_info *pbm); +void pci_get_pbm_props(struct pci_pbm_info *pbm); +struct pci_bus *pci_scan_one_pbm(struct pci_pbm_info *pbm, + struct device *parent); +void pci_determine_mem_io_space(struct pci_pbm_info *pbm); /* Error reporting support. */ -extern void pci_scan_for_target_abort(struct pci_pbm_info *, struct pci_bus *); -extern void pci_scan_for_master_abort(struct pci_pbm_info *, struct pci_bus *); -extern void pci_scan_for_parity_error(struct pci_pbm_info *, struct pci_bus *); +void pci_scan_for_target_abort(struct pci_pbm_info *, struct pci_bus *); +void pci_scan_for_master_abort(struct pci_pbm_info *, struct pci_bus *); +void pci_scan_for_parity_error(struct pci_pbm_info *, struct pci_bus *); /* Configuration space access. */ -extern void pci_config_read8(u8 *addr, u8 *ret); -extern void pci_config_read16(u16 *addr, u16 *ret); -extern void pci_config_read32(u32 *addr, u32 *ret); -extern void pci_config_write8(u8 *addr, u8 val); -extern void pci_config_write16(u16 *addr, u16 val); -extern void pci_config_write32(u32 *addr, u32 val); +void pci_config_read8(u8 *addr, u8 *ret); +void pci_config_read16(u16 *addr, u16 *ret); +void pci_config_read32(u32 *addr, u32 *ret); +void pci_config_write8(u8 *addr, u8 val); +void pci_config_write16(u16 *addr, u16 val); +void pci_config_write32(u32 *addr, u32 val); extern struct pci_ops sun4u_pci_ops; extern struct pci_ops sun4v_pci_ops; diff --git a/arch/sparc/kernel/pci_msi.c b/arch/sparc/kernel/pci_msi.c index 580651af73f2..acb2f83a1d5c 100644 --- a/arch/sparc/kernel/pci_msi.c +++ b/arch/sparc/kernel/pci_msi.c @@ -1,9 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 /* pci_msi.c: Sparc64 MSI support common layer. * * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ #include <linux/kernel.h> #include <linux/interrupt.h> +#include <linux/of.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/irq.h> @@ -111,10 +114,10 @@ static void free_msi(struct pci_pbm_info *pbm, int msi_num) static struct irq_chip msi_irq = { .name = "PCI-MSI", - .irq_mask = mask_msi_irq, - .irq_unmask = unmask_msi_irq, - .irq_enable = unmask_msi_irq, - .irq_disable = mask_msi_irq, + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, + .irq_enable = pci_msi_unmask_irq, + .irq_disable = pci_msi_mask_irq, /* XXX affinity XXX */ }; @@ -145,13 +148,13 @@ static int sparc64_setup_msi_irq(unsigned int *irq_p, msiqid = pick_msiq(pbm); err = ops->msi_setup(pbm, msiqid, msi, - (entry->msi_attrib.is_64 ? 1 : 0)); + (entry->pci.msi_attrib.is_64 ? 1 : 0)); if (err) goto out_msi_free; pbm->msi_irq_table[msi - pbm->msi_first] = *irq_p; - if (entry->msi_attrib.is_64) { + if (entry->pci.msi_attrib.is_64) { msg.address_hi = pbm->msi64_start >> 32; msg.address_lo = pbm->msi64_start & 0xffffffff; } else { @@ -161,7 +164,7 @@ static int sparc64_setup_msi_irq(unsigned int *irq_p, msg.data = msi; irq_set_msi_desc(*irq_p, entry); - write_msi_msg(*irq_p, &msg); + pci_write_msi_msg(*irq_p, &msg); return 0; @@ -190,8 +193,8 @@ static void sparc64_teardown_msi_irq(unsigned int irq, break; } if (i >= pbm->msi_num) { - printk(KERN_ERR "%s: teardown: No MSI for irq %u\n", - pbm->name, irq); + pci_err(pdev, "%s: teardown: No MSI for irq %u\n", pbm->name, + irq); return; } @@ -200,9 +203,9 @@ static void sparc64_teardown_msi_irq(unsigned int irq, err = ops->msi_teardown(pbm, msi_num); if (err) { - printk(KERN_ERR "%s: teardown: ops->teardown() on MSI %u, " - "irq %u, gives error %d\n", - pbm->name, msi_num, irq, err); + pci_err(pdev, "%s: teardown: ops->teardown() on MSI %u, " + "irq %u, gives error %d\n", pbm->name, msi_num, irq, + err); return; } @@ -284,10 +287,7 @@ static int bringup_one_msi_queue(struct pci_pbm_info *pbm, nid = pbm->numa_node; if (nid != -1) { - cpumask_t numa_mask; - - cpumask_copy(&numa_mask, cpumask_of_node(nid)); - irq_set_affinity(irq, &numa_mask); + irq_set_affinity(irq, cpumask_of_node(nid)); } err = request_irq(irq, sparc64_msiq_interrupt, 0, "MSIQ", diff --git a/arch/sparc/kernel/pci_psycho.c b/arch/sparc/kernel/pci_psycho.c index c647634ead2b..1efc98305ec7 100644 --- a/arch/sparc/kernel/pci_psycho.c +++ b/arch/sparc/kernel/pci_psycho.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* pci_psycho.c: PSYCHO/U2P specific PCI controller support. * * Copyright (C) 1997, 1998, 1999, 2007 David S. Miller (davem@davemloft.net) @@ -12,7 +13,9 @@ #include <linux/export.h> #include <linux/slab.h> #include <linux/interrupt.h> -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <asm/iommu.h> #include <asm/irq.h> @@ -604,7 +607,6 @@ static const struct of_device_id psycho_match[] = { static struct platform_driver psycho_driver = { .driver = { .name = DRIVER_NAME, - .owner = THIS_MODULE, .of_match_table = psycho_match, }, .probe = psycho_probe, diff --git a/arch/sparc/kernel/pci_sabre.c b/arch/sparc/kernel/pci_sabre.c index 6f00d27e8dac..a84598568300 100644 --- a/arch/sparc/kernel/pci_sabre.c +++ b/arch/sparc/kernel/pci_sabre.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* pci_sabre.c: Sabre specific PCI controller support. * * Copyright (C) 1997, 1998, 1999, 2007 David S. Miller (davem@davemloft.net) @@ -12,7 +13,10 @@ #include <linux/export.h> #include <linux/slab.h> #include <linux/interrupt.h> -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/property.h> #include <asm/apb.h> #include <asm/iommu.h> @@ -455,7 +459,6 @@ static void sabre_pbm_init(struct pci_pbm_info *pbm, static const struct of_device_id sabre_match[]; static int sabre_probe(struct platform_device *op) { - const struct of_device_id *match; const struct linux_prom64_registers *pr_regs; struct device_node *dp = op->dev.of_node; struct pci_pbm_info *pbm; @@ -465,8 +468,7 @@ static int sabre_probe(struct platform_device *op) const u32 *vdma; u64 clear_irq; - match = of_match_device(sabre_match, &op->dev); - hummingbird_p = match && (match->data != NULL); + hummingbird_p = (uintptr_t)device_get_match_data(&op->dev); if (!hummingbird_p) { struct device_node *cpu_dp; @@ -474,7 +476,7 @@ static int sabre_probe(struct platform_device *op) * different ways, inconsistently. */ for_each_node_by_type(cpu_dp, "cpu") { - if (!strcmp(cpu_dp->name, "SUNW,UltraSPARC-IIe")) + if (of_node_name_eq(cpu_dp, "SUNW,UltraSPARC-IIe")) hummingbird_p = 1; } } @@ -600,7 +602,6 @@ static const struct of_device_id sabre_match[] = { static struct platform_driver sabre_driver = { .driver = { .name = DRIVER_NAME, - .owner = THIS_MODULE, .of_match_table = sabre_match, }, .probe = sabre_probe, diff --git a/arch/sparc/kernel/pci_schizo.c b/arch/sparc/kernel/pci_schizo.c index 8f76f23dac38..93cd9e5a8099 100644 --- a/arch/sparc/kernel/pci_schizo.c +++ b/arch/sparc/kernel/pci_schizo.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* pci_schizo.c: SCHIZO/TOMATILLO specific PCI controller support. * * Copyright (C) 2001, 2002, 2003, 2007, 2008 David S. Miller (davem@davemloft.net) @@ -10,7 +11,11 @@ #include <linux/slab.h> #include <linux/export.h> #include <linux/interrupt.h> -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/property.h> +#include <linux/numa.h> #include <asm/iommu.h> #include <asm/irq.h> @@ -140,7 +145,7 @@ static void __schizo_check_stc_error_pbm(struct pci_pbm_info *pbm, /* This is __REALLY__ dangerous. When we put the * streaming buffer into diagnostic mode to probe - * it's tags and error status, we _must_ clear all + * its tags and error status, we _must_ clear all * of the line tag valid bits before re-enabling * the streaming buffer. If any dirty data lives * in the STC when we do this, we will end up @@ -270,7 +275,7 @@ static void schizo_check_iommu_error_pbm(struct pci_pbm_info *pbm, pbm->name, type_string); /* Put the IOMMU into diagnostic mode and probe - * it's TLB for entries with error status. + * its TLB for entries with error status. * * It is very possible for another DVMA to occur * while we do this probe, and corrupt the system @@ -581,7 +586,7 @@ static irqreturn_t schizo_pcierr_intr_other(struct pci_pbm_info *pbm) { unsigned long csr_reg, csr, csr_error_bits; irqreturn_t ret = IRQ_NONE; - u16 stat; + u32 stat; csr_reg = pbm->pbm_regs + SCHIZO_PCI_CTRL; csr = upa_readq(csr_reg); @@ -617,7 +622,7 @@ static irqreturn_t schizo_pcierr_intr_other(struct pci_pbm_info *pbm) pbm->name); ret = IRQ_HANDLED; } - pci_read_config_word(pbm->pci_bus->self, PCI_STATUS, &stat); + pbm->pci_ops->read(pbm->pci_bus, 0, PCI_STATUS, 2, &stat); if (stat & (PCI_STATUS_PARITY | PCI_STATUS_SIG_TARGET_ABORT | PCI_STATUS_REC_TARGET_ABORT | @@ -625,7 +630,7 @@ static irqreturn_t schizo_pcierr_intr_other(struct pci_pbm_info *pbm) PCI_STATUS_SIG_SYSTEM_ERROR)) { printk("%s: PCI bus error, PCI_STATUS[%04x]\n", pbm->name, stat); - pci_write_config_word(pbm->pci_bus->self, PCI_STATUS, 0xffff); + pbm->pci_ops->write(pbm->pci_bus, 0, PCI_STATUS, 2, 0xffff); ret = IRQ_HANDLED; } return ret; @@ -1268,7 +1273,7 @@ static void schizo_pbm_hw_init(struct pci_pbm_info *pbm) pbm->chip_version >= 0x2) tmp |= 0x3UL << SCHIZO_PCICTRL_PTO_SHIFT; - if (!of_find_property(pbm->op->dev.of_node, "no-bus-parking", NULL)) + if (!of_property_read_bool(pbm->op->dev.of_node, "no-bus-parking")) tmp |= SCHIZO_PCICTRL_PARK; else tmp &= ~SCHIZO_PCICTRL_PARK; @@ -1346,7 +1351,7 @@ static int schizo_pbm_init(struct pci_pbm_info *pbm, pbm->next = pci_pbm_root; pci_pbm_root = pbm; - pbm->numa_node = -1; + pbm->numa_node = NUMA_NO_NODE; pbm->pci_ops = &sun4u_pci_ops; pbm->config_space_reg_bits = 8; @@ -1457,15 +1462,13 @@ out_err: return err; } -static const struct of_device_id schizo_match[]; static int schizo_probe(struct platform_device *op) { - const struct of_device_id *match; + unsigned long chip_type = (unsigned long)device_get_match_data(&op->dev); - match = of_match_device(schizo_match, &op->dev); - if (!match) + if (!chip_type) return -EINVAL; - return __schizo_init(op, (unsigned long)match->data); + return __schizo_init(op, chip_type); } /* The ordering of this table is very important. Some Tomatillo @@ -1495,7 +1498,6 @@ static const struct of_device_id schizo_match[] = { static struct platform_driver schizo_driver = { .driver = { .name = DRIVER_NAME, - .owner = THIS_MODULE, .of_match_table = schizo_match, }, .probe = schizo_probe, diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c index d07f6b29aed8..791f0a76665f 100644 --- a/arch/sparc/kernel/pci_sun4v.c +++ b/arch/sparc/kernel/pci_sun4v.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* pci_sun4v.c: SUN4V specific PCI controller support. * * Copyright (C) 2006, 2007, 2008 David S. Miller (davem@davemloft.net) @@ -14,7 +15,10 @@ #include <linux/msi.h> #include <linux/export.h> #include <linux/log2.h> -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/dma-map-ops.h> +#include <asm/iommu-common.h> #include <asm/iommu.h> #include <asm/irq.h> @@ -23,14 +27,29 @@ #include "pci_impl.h" #include "iommu_common.h" +#include "kernel.h" #include "pci_sun4v.h" #define DRIVER_NAME "pci_sun4v" #define PFX DRIVER_NAME ": " -static unsigned long vpci_major = 1; -static unsigned long vpci_minor = 1; +static unsigned long vpci_major; +static unsigned long vpci_minor; + +struct vpci_version { + unsigned long major; + unsigned long minor; +}; + +/* Ordered from largest major to lowest */ +static struct vpci_version vpci_versions[] = { + { .major = 2, .minor = 0 }, + { .major = 1, .minor = 1 }, +}; + +static unsigned long vatu_major = 1; +static unsigned long vatu_minor = 1; #define PGLIST_NENTS (PAGE_SIZE / sizeof(u64)) @@ -48,7 +67,7 @@ static int iommu_batch_initialized; /* Interrupts must be disabled. */ static inline void iommu_batch_start(struct device *dev, unsigned long prot, unsigned long entry) { - struct iommu_batch *p = &__get_cpu_var(iommu_batch); + struct iommu_batch *p = this_cpu_ptr(&iommu_batch); p->dev = dev; p->prot = prot; @@ -56,31 +75,63 @@ static inline void iommu_batch_start(struct device *dev, unsigned long prot, uns p->npages = 0; } +static inline bool iommu_use_atu(struct iommu *iommu, u64 mask) +{ + return iommu->atu && mask > DMA_BIT_MASK(32); +} + /* Interrupts must be disabled. */ -static long iommu_batch_flush(struct iommu_batch *p) +static long iommu_batch_flush(struct iommu_batch *p, u64 mask) { struct pci_pbm_info *pbm = p->dev->archdata.host_controller; + u64 *pglist = p->pglist; + u64 index_count; unsigned long devhandle = pbm->devhandle; unsigned long prot = p->prot; unsigned long entry = p->entry; - u64 *pglist = p->pglist; unsigned long npages = p->npages; + unsigned long iotsb_num; + unsigned long ret; + long num; + + /* VPCI maj=1, min=[0,1] only supports read and write */ + if (vpci_major < 2) + prot &= (HV_PCI_MAP_ATTR_READ | HV_PCI_MAP_ATTR_WRITE); while (npages != 0) { - long num; - - num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry), - npages, prot, __pa(pglist)); - if (unlikely(num < 0)) { - if (printk_ratelimit()) - printk("iommu_batch_flush: IOMMU map of " - "[%08lx:%08llx:%lx:%lx:%lx] failed with " - "status %ld\n", - devhandle, HV_PCI_TSBID(0, entry), - npages, prot, __pa(pglist), num); - return -1; + if (!iommu_use_atu(pbm->iommu, mask)) { + num = pci_sun4v_iommu_map(devhandle, + HV_PCI_TSBID(0, entry), + npages, + prot, + __pa(pglist)); + if (unlikely(num < 0)) { + pr_err_ratelimited("%s: IOMMU map of [%08lx:%08llx:%lx:%lx:%lx] failed with status %ld\n", + __func__, + devhandle, + HV_PCI_TSBID(0, entry), + npages, prot, __pa(pglist), + num); + return -1; + } + } else { + index_count = HV_PCI_IOTSB_INDEX_COUNT(npages, entry), + iotsb_num = pbm->iommu->atu->iotsb->iotsb_num; + ret = pci_sun4v_iotsb_map(devhandle, + iotsb_num, + index_count, + prot, + __pa(pglist), + &num); + if (unlikely(ret != HV_EOK)) { + pr_err_ratelimited("%s: ATU map of [%08lx:%lx:%llx:%lx:%lx] failed with status %ld\n", + __func__, + devhandle, iotsb_num, + index_count, prot, + __pa(pglist), ret); + return -1; + } } - entry += num; npages -= num; pglist += num; @@ -92,47 +143,50 @@ static long iommu_batch_flush(struct iommu_batch *p) return 0; } -static inline void iommu_batch_new_entry(unsigned long entry) +static inline void iommu_batch_new_entry(unsigned long entry, u64 mask) { - struct iommu_batch *p = &__get_cpu_var(iommu_batch); + struct iommu_batch *p = this_cpu_ptr(&iommu_batch); if (p->entry + p->npages == entry) return; if (p->entry != ~0UL) - iommu_batch_flush(p); + iommu_batch_flush(p, mask); p->entry = entry; } /* Interrupts must be disabled. */ -static inline long iommu_batch_add(u64 phys_page) +static inline long iommu_batch_add(u64 phys_page, u64 mask) { - struct iommu_batch *p = &__get_cpu_var(iommu_batch); + struct iommu_batch *p = this_cpu_ptr(&iommu_batch); BUG_ON(p->npages >= PGLIST_NENTS); p->pglist[p->npages++] = phys_page; if (p->npages == PGLIST_NENTS) - return iommu_batch_flush(p); + return iommu_batch_flush(p, mask); return 0; } /* Interrupts must be disabled. */ -static inline long iommu_batch_end(void) +static inline long iommu_batch_end(u64 mask) { - struct iommu_batch *p = &__get_cpu_var(iommu_batch); + struct iommu_batch *p = this_cpu_ptr(&iommu_batch); BUG_ON(p->npages >= PGLIST_NENTS); - return iommu_batch_flush(p); + return iommu_batch_flush(p, mask); } static void *dma_4v_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { + u64 mask; unsigned long flags, order, first_page, npages, n; + unsigned long prot = 0; struct iommu *iommu; + struct iommu_map_table *tbl; struct page *page; void *ret; long entry; @@ -140,11 +194,14 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size, size = IO_PAGE_ALIGN(size); order = get_order(size); - if (unlikely(order >= MAX_ORDER)) + if (unlikely(order > MAX_PAGE_ORDER)) return NULL; npages = size >> IO_PAGE_SHIFT; + if (attrs & DMA_ATTR_WEAK_ORDERING) + prot = HV_PCI_MAP_ATTR_RELAXED_ORDER; + nid = dev->archdata.numa_node; page = alloc_pages_node(nid, gfp, order); if (unlikely(!page)) @@ -154,33 +211,36 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size, memset((char *)first_page, 0, PAGE_SIZE << order); iommu = dev->archdata.iommu; + mask = dev->coherent_dma_mask; + if (!iommu_use_atu(iommu, mask)) + tbl = &iommu->tbl; + else + tbl = &iommu->atu->tbl; - spin_lock_irqsave(&iommu->lock, flags); - entry = iommu_range_alloc(dev, iommu, npages, NULL); - spin_unlock_irqrestore(&iommu->lock, flags); + entry = iommu_tbl_range_alloc(dev, tbl, npages, NULL, + (unsigned long)(-1), 0); - if (unlikely(entry == DMA_ERROR_CODE)) + if (unlikely(entry == IOMMU_ERROR_CODE)) goto range_alloc_fail; - *dma_addrp = (iommu->page_table_map_base + - (entry << IO_PAGE_SHIFT)); + *dma_addrp = (tbl->table_map_base + (entry << IO_PAGE_SHIFT)); ret = (void *) first_page; first_page = __pa(first_page); local_irq_save(flags); iommu_batch_start(dev, - (HV_PCI_MAP_ATTR_READ | + (HV_PCI_MAP_ATTR_READ | prot | HV_PCI_MAP_ATTR_WRITE), entry); for (n = 0; n < npages; n++) { - long err = iommu_batch_add(first_page + (n * PAGE_SIZE)); + long err = iommu_batch_add(first_page + (n * PAGE_SIZE), mask); if (unlikely(err < 0L)) goto iommu_map_fail; } - if (unlikely(iommu_batch_end() < 0L)) + if (unlikely(iommu_batch_end(mask) < 0L)) goto iommu_map_fail; local_irq_restore(flags); @@ -188,96 +248,174 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size, return ret; iommu_map_fail: - /* Interrupts are disabled. */ - spin_lock(&iommu->lock); - iommu_range_free(iommu, *dma_addrp, npages); - spin_unlock_irqrestore(&iommu->lock, flags); + local_irq_restore(flags); + iommu_tbl_range_free(tbl, *dma_addrp, npages, IOMMU_ERROR_CODE); range_alloc_fail: free_pages(first_page, order); return NULL; } +static unsigned long dma_4v_iotsb_bind(unsigned long devhandle, + unsigned long iotsb_num, + struct pci_bus *bus_dev) +{ + struct pci_dev *pdev; + unsigned long err; + unsigned int bus; + unsigned int device; + unsigned int fun; + + list_for_each_entry(pdev, &bus_dev->devices, bus_list) { + if (pdev->subordinate) { + /* No need to bind pci bridge */ + dma_4v_iotsb_bind(devhandle, iotsb_num, + pdev->subordinate); + } else { + bus = bus_dev->number; + device = PCI_SLOT(pdev->devfn); + fun = PCI_FUNC(pdev->devfn); + err = pci_sun4v_iotsb_bind(devhandle, iotsb_num, + HV_PCI_DEVICE_BUILD(bus, + device, + fun)); + + /* If bind fails for one device it is going to fail + * for rest of the devices because we are sharing + * IOTSB. So in case of failure simply return with + * error. + */ + if (err) + return err; + } + } + + return 0; +} + +static void dma_4v_iommu_demap(struct device *dev, unsigned long devhandle, + dma_addr_t dvma, unsigned long iotsb_num, + unsigned long entry, unsigned long npages) +{ + unsigned long num, flags; + unsigned long ret; + + local_irq_save(flags); + do { + if (dvma <= DMA_BIT_MASK(32)) { + num = pci_sun4v_iommu_demap(devhandle, + HV_PCI_TSBID(0, entry), + npages); + } else { + ret = pci_sun4v_iotsb_demap(devhandle, iotsb_num, + entry, npages, &num); + if (unlikely(ret != HV_EOK)) { + pr_err_ratelimited("pci_iotsb_demap() failed with error: %ld\n", + ret); + } + } + entry += num; + npages -= num; + } while (npages != 0); + local_irq_restore(flags); +} + static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, - dma_addr_t dvma, struct dma_attrs *attrs) + dma_addr_t dvma, unsigned long attrs) { struct pci_pbm_info *pbm; struct iommu *iommu; - unsigned long flags, order, npages, entry; + struct atu *atu; + struct iommu_map_table *tbl; + unsigned long order, npages, entry; + unsigned long iotsb_num; u32 devhandle; npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; iommu = dev->archdata.iommu; pbm = dev->archdata.host_controller; + atu = iommu->atu; devhandle = pbm->devhandle; - entry = ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT); - - spin_lock_irqsave(&iommu->lock, flags); - - iommu_range_free(iommu, dvma, npages); - - do { - unsigned long num; - - num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), - npages); - entry += num; - npages -= num; - } while (npages != 0); - - spin_unlock_irqrestore(&iommu->lock, flags); + if (!iommu_use_atu(iommu, dvma)) { + tbl = &iommu->tbl; + iotsb_num = 0; /* we don't care for legacy iommu */ + } else { + tbl = &atu->tbl; + iotsb_num = atu->iotsb->iotsb_num; + } + entry = ((dvma - tbl->table_map_base) >> IO_PAGE_SHIFT); + dma_4v_iommu_demap(dev, devhandle, dvma, iotsb_num, entry, npages); + iommu_tbl_range_free(tbl, dvma, npages, IOMMU_ERROR_CODE); order = get_order(size); if (order < 10) free_pages((unsigned long)cpu, order); } -static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t sz, - enum dma_data_direction direction, - struct dma_attrs *attrs) +static dma_addr_t dma_4v_map_phys(struct device *dev, phys_addr_t phys, + size_t sz, enum dma_data_direction direction, + unsigned long attrs) { struct iommu *iommu; + struct atu *atu; + struct iommu_map_table *tbl; + u64 mask; unsigned long flags, npages, oaddr; - unsigned long i, base_paddr; - u32 bus_addr, ret; - unsigned long prot; + unsigned long i, prot; + dma_addr_t bus_addr, ret; long entry; + if (unlikely(attrs & DMA_ATTR_MMIO)) + /* + * This check is included because older versions of the code + * lacked MMIO path support, and my ability to test this path + * is limited. However, from a software technical standpoint, + * there is no restriction, as the following code operates + * solely on physical addresses. + */ + goto bad; + iommu = dev->archdata.iommu; + atu = iommu->atu; if (unlikely(direction == DMA_NONE)) goto bad; - oaddr = (unsigned long)(page_address(page) + offset); + oaddr = (unsigned long)(phys_to_virt(phys)); npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); npages >>= IO_PAGE_SHIFT; - spin_lock_irqsave(&iommu->lock, flags); - entry = iommu_range_alloc(dev, iommu, npages, NULL); - spin_unlock_irqrestore(&iommu->lock, flags); + mask = *dev->dma_mask; + if (!iommu_use_atu(iommu, mask)) + tbl = &iommu->tbl; + else + tbl = &atu->tbl; + + entry = iommu_tbl_range_alloc(dev, tbl, npages, NULL, + (unsigned long)(-1), 0); - if (unlikely(entry == DMA_ERROR_CODE)) + if (unlikely(entry == IOMMU_ERROR_CODE)) goto bad; - bus_addr = (iommu->page_table_map_base + - (entry << IO_PAGE_SHIFT)); + bus_addr = (tbl->table_map_base + (entry << IO_PAGE_SHIFT)); ret = bus_addr | (oaddr & ~IO_PAGE_MASK); - base_paddr = __pa(oaddr & IO_PAGE_MASK); prot = HV_PCI_MAP_ATTR_READ; if (direction != DMA_TO_DEVICE) prot |= HV_PCI_MAP_ATTR_WRITE; + if (attrs & DMA_ATTR_WEAK_ORDERING) + prot |= HV_PCI_MAP_ATTR_RELAXED_ORDER; + local_irq_save(flags); iommu_batch_start(dev, prot, entry); - for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) { - long err = iommu_batch_add(base_paddr); + for (i = 0; i < npages; i++, phys += IO_PAGE_SIZE) { + long err = iommu_batch_add(phys, mask); if (unlikely(err < 0L)) goto iommu_map_fail; } - if (unlikely(iommu_batch_end() < 0L)) + if (unlikely(iommu_batch_end(mask) < 0L)) goto iommu_map_fail; local_irq_restore(flags); @@ -287,24 +425,24 @@ static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, bad: if (printk_ratelimit()) WARN_ON(1); - return DMA_ERROR_CODE; + return DMA_MAPPING_ERROR; iommu_map_fail: - /* Interrupts are disabled. */ - spin_lock(&iommu->lock); - iommu_range_free(iommu, bus_addr, npages); - spin_unlock_irqrestore(&iommu->lock, flags); - - return DMA_ERROR_CODE; + local_irq_restore(flags); + iommu_tbl_range_free(tbl, bus_addr, npages, IOMMU_ERROR_CODE); + return DMA_MAPPING_ERROR; } -static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr, +static void dma_4v_unmap_phys(struct device *dev, dma_addr_t bus_addr, size_t sz, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct pci_pbm_info *pbm; struct iommu *iommu; - unsigned long flags, npages; + struct atu *atu; + struct iommu_map_table *tbl; + unsigned long npages; + unsigned long iotsb_num; long entry; u32 devhandle; @@ -316,32 +454,28 @@ static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr, iommu = dev->archdata.iommu; pbm = dev->archdata.host_controller; + atu = iommu->atu; devhandle = pbm->devhandle; npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); npages >>= IO_PAGE_SHIFT; bus_addr &= IO_PAGE_MASK; - spin_lock_irqsave(&iommu->lock, flags); - - iommu_range_free(iommu, bus_addr, npages); - - entry = (bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT; - do { - unsigned long num; - - num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), - npages); - entry += num; - npages -= num; - } while (npages != 0); - - spin_unlock_irqrestore(&iommu->lock, flags); + if (bus_addr <= DMA_BIT_MASK(32)) { + iotsb_num = 0; /* we don't care for legacy iommu */ + tbl = &iommu->tbl; + } else { + iotsb_num = atu->iotsb->iotsb_num; + tbl = &atu->tbl; + } + entry = (bus_addr - tbl->table_map_base) >> IO_PAGE_SHIFT; + dma_4v_iommu_demap(dev, devhandle, bus_addr, iotsb_num, entry, npages); + iommu_tbl_range_free(tbl, bus_addr, npages, IOMMU_ERROR_CODE); } static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s, *outs, *segstart; unsigned long flags, handle, prot; @@ -350,6 +484,9 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, unsigned long seg_boundary_size; int outcount, incount, i; struct iommu *iommu; + struct atu *atu; + struct iommu_map_table *tbl; + u64 mask; unsigned long base_shift; long err; @@ -357,12 +494,16 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, iommu = dev->archdata.iommu; if (nelems == 0 || !iommu) - return 0; - + return -EINVAL; + atu = iommu->atu; + prot = HV_PCI_MAP_ATTR_READ; if (direction != DMA_TO_DEVICE) prot |= HV_PCI_MAP_ATTR_WRITE; + if (attrs & DMA_ATTR_WEAK_ORDERING) + prot |= HV_PCI_MAP_ATTR_RELAXED_ORDER; + outs = s = segstart = &sglist[0]; outcount = 1; incount = nelems; @@ -371,14 +512,21 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, /* Init first segment length for backout at failure */ outs->dma_length = 0; - spin_lock_irqsave(&iommu->lock, flags); + local_irq_save(flags); iommu_batch_start(dev, prot, ~0UL); max_seg_size = dma_get_max_seg_size(dev); - seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, - IO_PAGE_SIZE) >> IO_PAGE_SHIFT; - base_shift = iommu->page_table_map_base >> IO_PAGE_SHIFT; + seg_boundary_size = dma_get_seg_boundary_nr_pages(dev, IO_PAGE_SHIFT); + + mask = *dev->dma_mask; + if (!iommu_use_atu(iommu, mask)) + tbl = &iommu->tbl; + else + tbl = &atu->tbl; + + base_shift = tbl->table_map_base >> IO_PAGE_SHIFT; + for_each_sg(sglist, s, nelems, i) { unsigned long paddr, npages, entry, out_entry = 0, slen; @@ -391,27 +539,26 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, /* Allocate iommu entries for that segment */ paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); - entry = iommu_range_alloc(dev, iommu, npages, &handle); + entry = iommu_tbl_range_alloc(dev, tbl, npages, + &handle, (unsigned long)(-1), 0); /* Handle failure */ - if (unlikely(entry == DMA_ERROR_CODE)) { - if (printk_ratelimit()) - printk(KERN_INFO "iommu_alloc failed, iommu %p paddr %lx" - " npages %lx\n", iommu, paddr, npages); + if (unlikely(entry == IOMMU_ERROR_CODE)) { + pr_err_ratelimited("iommu_alloc failed, iommu %p paddr %lx npages %lx\n", + tbl, paddr, npages); goto iommu_map_failed; } - iommu_batch_new_entry(entry); + iommu_batch_new_entry(entry, mask); /* Convert entry to a dma_addr_t */ - dma_addr = iommu->page_table_map_base + - (entry << IO_PAGE_SHIFT); + dma_addr = tbl->table_map_base + (entry << IO_PAGE_SHIFT); dma_addr |= (s->offset & ~IO_PAGE_MASK); /* Insert into HW table */ paddr &= IO_PAGE_MASK; while (npages--) { - err = iommu_batch_add(paddr); + err = iommu_batch_add(paddr, mask); if (unlikely(err < 0L)) goto iommu_map_failed; paddr += IO_PAGE_SIZE; @@ -446,16 +593,15 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, dma_next = dma_addr + slen; } - err = iommu_batch_end(); + err = iommu_batch_end(mask); if (unlikely(err < 0L)) goto iommu_map_failed; - spin_unlock_irqrestore(&iommu->lock, flags); + local_irq_restore(flags); if (outcount < incount) { outs = sg_next(outs); - outs->dma_address = DMA_ERROR_CODE; outs->dma_length = 0; } @@ -469,71 +615,89 @@ iommu_map_failed: vaddr = s->dma_address & IO_PAGE_MASK; npages = iommu_num_pages(s->dma_address, s->dma_length, IO_PAGE_SIZE); - iommu_range_free(iommu, vaddr, npages); + iommu_tbl_range_free(tbl, vaddr, npages, + IOMMU_ERROR_CODE); /* XXX demap? XXX */ - s->dma_address = DMA_ERROR_CODE; s->dma_length = 0; } if (s == outs) break; } - spin_unlock_irqrestore(&iommu->lock, flags); + local_irq_restore(flags); - return 0; + return -EINVAL; } static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct pci_pbm_info *pbm; struct scatterlist *sg; struct iommu *iommu; - unsigned long flags; + struct atu *atu; + unsigned long flags, entry; + unsigned long iotsb_num; u32 devhandle; BUG_ON(direction == DMA_NONE); iommu = dev->archdata.iommu; pbm = dev->archdata.host_controller; + atu = iommu->atu; devhandle = pbm->devhandle; - spin_lock_irqsave(&iommu->lock, flags); + local_irq_save(flags); sg = sglist; while (nelems--) { dma_addr_t dma_handle = sg->dma_address; unsigned int len = sg->dma_length; - unsigned long npages, entry; + unsigned long npages; + struct iommu_map_table *tbl; + unsigned long shift = IO_PAGE_SHIFT; if (!len) break; npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); - iommu_range_free(iommu, dma_handle, npages); - entry = ((dma_handle - iommu->page_table_map_base) >> IO_PAGE_SHIFT); - while (npages) { - unsigned long num; - - num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), - npages); - entry += num; - npages -= num; + if (dma_handle <= DMA_BIT_MASK(32)) { + iotsb_num = 0; /* we don't care for legacy iommu */ + tbl = &iommu->tbl; + } else { + iotsb_num = atu->iotsb->iotsb_num; + tbl = &atu->tbl; } - + entry = ((dma_handle - tbl->table_map_base) >> shift); + dma_4v_iommu_demap(dev, devhandle, dma_handle, iotsb_num, + entry, npages); + iommu_tbl_range_free(tbl, dma_handle, npages, + IOMMU_ERROR_CODE); sg = sg_next(sg); } - spin_unlock_irqrestore(&iommu->lock, flags); + local_irq_restore(flags); +} + +static int dma_4v_supported(struct device *dev, u64 device_mask) +{ + struct iommu *iommu = dev->archdata.iommu; + + if (ali_sound_dma_hack(dev, device_mask)) + return 1; + if (device_mask < iommu->dma_addr_mask) + return 0; + return 1; } -static struct dma_map_ops sun4v_dma_ops = { +static const struct dma_map_ops sun4v_dma_ops = { .alloc = dma_4v_alloc_coherent, .free = dma_4v_free_coherent, - .map_page = dma_4v_map_page, - .unmap_page = dma_4v_unmap_page, + .map_phys = dma_4v_map_phys, + .unmap_phys = dma_4v_unmap_phys, .map_sg = dma_4v_map_sg, .unmap_sg = dma_4v_unmap_sg, + .dma_supported = dma_4v_supported, }; static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent) @@ -550,33 +714,162 @@ static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent) } static unsigned long probe_existing_entries(struct pci_pbm_info *pbm, - struct iommu *iommu) + struct iommu_map_table *iommu) { - struct iommu_arena *arena = &iommu->arena; - unsigned long i, cnt = 0; + struct iommu_pool *pool; + unsigned long i, pool_nr, cnt = 0; u32 devhandle; devhandle = pbm->devhandle; - for (i = 0; i < arena->limit; i++) { - unsigned long ret, io_attrs, ra; - - ret = pci_sun4v_iommu_getmap(devhandle, - HV_PCI_TSBID(0, i), - &io_attrs, &ra); - if (ret == HV_EOK) { - if (page_in_phys_avail(ra)) { - pci_sun4v_iommu_demap(devhandle, - HV_PCI_TSBID(0, i), 1); - } else { - cnt++; - __set_bit(i, arena->map); + for (pool_nr = 0; pool_nr < iommu->nr_pools; pool_nr++) { + pool = &(iommu->pools[pool_nr]); + for (i = pool->start; i <= pool->end; i++) { + unsigned long ret, io_attrs, ra; + + ret = pci_sun4v_iommu_getmap(devhandle, + HV_PCI_TSBID(0, i), + &io_attrs, &ra); + if (ret == HV_EOK) { + if (page_in_phys_avail(ra)) { + pci_sun4v_iommu_demap(devhandle, + HV_PCI_TSBID(0, + i), 1); + } else { + cnt++; + __set_bit(i, iommu->map); + } } } } - return cnt; } +static int pci_sun4v_atu_alloc_iotsb(struct pci_pbm_info *pbm) +{ + struct atu *atu = pbm->iommu->atu; + struct atu_iotsb *iotsb; + void *table; + u64 table_size; + u64 iotsb_num; + unsigned long order; + unsigned long err; + + iotsb = kzalloc(sizeof(*iotsb), GFP_KERNEL); + if (!iotsb) { + err = -ENOMEM; + goto out_err; + } + atu->iotsb = iotsb; + + /* calculate size of IOTSB */ + table_size = (atu->size / IO_PAGE_SIZE) * 8; + order = get_order(table_size); + table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + if (!table) { + err = -ENOMEM; + goto table_failed; + } + iotsb->table = table; + iotsb->ra = __pa(table); + iotsb->dvma_size = atu->size; + iotsb->dvma_base = atu->base; + iotsb->table_size = table_size; + iotsb->page_size = IO_PAGE_SIZE; + + /* configure and register IOTSB with HV */ + err = pci_sun4v_iotsb_conf(pbm->devhandle, + iotsb->ra, + iotsb->table_size, + iotsb->page_size, + iotsb->dvma_base, + &iotsb_num); + if (err) { + pr_err(PFX "pci_iotsb_conf failed error: %ld\n", err); + goto iotsb_conf_failed; + } + iotsb->iotsb_num = iotsb_num; + + err = dma_4v_iotsb_bind(pbm->devhandle, iotsb_num, pbm->pci_bus); + if (err) { + pr_err(PFX "pci_iotsb_bind failed error: %ld\n", err); + goto iotsb_conf_failed; + } + + return 0; + +iotsb_conf_failed: + free_pages((unsigned long)table, order); +table_failed: + kfree(iotsb); +out_err: + return err; +} + +static int pci_sun4v_atu_init(struct pci_pbm_info *pbm) +{ + struct atu *atu = pbm->iommu->atu; + unsigned long err; + const u64 *ranges; + u64 map_size, num_iotte; + u64 dma_mask; + const u32 *page_size; + int len; + + ranges = of_get_property(pbm->op->dev.of_node, "iommu-address-ranges", + &len); + if (!ranges) { + pr_err(PFX "No iommu-address-ranges\n"); + return -EINVAL; + } + + page_size = of_get_property(pbm->op->dev.of_node, "iommu-pagesizes", + NULL); + if (!page_size) { + pr_err(PFX "No iommu-pagesizes\n"); + return -EINVAL; + } + + /* There are 4 iommu-address-ranges supported. Each range is pair of + * {base, size}. The ranges[0] and ranges[1] are 32bit address space + * while ranges[2] and ranges[3] are 64bit space. We want to use 64bit + * address ranges to support 64bit addressing. Because 'size' for + * address ranges[2] and ranges[3] are same we can select either of + * ranges[2] or ranges[3] for mapping. However due to 'size' is too + * large for OS to allocate IOTSB we are using fix size 32G + * (ATU_64_SPACE_SIZE) which is more than enough for all PCIe devices + * to share. + */ + atu->ranges = (struct atu_ranges *)ranges; + atu->base = atu->ranges[3].base; + atu->size = ATU_64_SPACE_SIZE; + + /* Create IOTSB */ + err = pci_sun4v_atu_alloc_iotsb(pbm); + if (err) { + pr_err(PFX "Error creating ATU IOTSB\n"); + return err; + } + + /* Create ATU iommu map. + * One bit represents one iotte in IOTSB table. + */ + dma_mask = (roundup_pow_of_two(atu->size) - 1UL); + num_iotte = atu->size / IO_PAGE_SIZE; + map_size = num_iotte / 8; + atu->tbl.table_map_base = atu->base; + atu->dma_addr_mask = dma_mask; + atu->tbl.map = kzalloc(map_size, GFP_KERNEL); + if (!atu->tbl.map) + return -ENOMEM; + + iommu_tbl_pool_init(&atu->tbl, num_iotte, IO_PAGE_SHIFT, + NULL, false /* no large_pool */, + 0 /* default npools */, + false /* want span boundary checking */); + + return 0; +} + static int pci_sun4v_iommu_init(struct pci_pbm_info *pbm) { static const u32 vdma_default[] = { 0x80000000, 0x80000000 }; @@ -603,20 +896,22 @@ static int pci_sun4v_iommu_init(struct pci_pbm_info *pbm) /* Setup initial software IOMMU state. */ spin_lock_init(&iommu->lock); iommu->ctx_lowest_free = 1; - iommu->page_table_map_base = dma_offset; + iommu->tbl.table_map_base = dma_offset; iommu->dma_addr_mask = dma_mask; /* Allocate and initialize the free area map. */ sz = (num_tsb_entries + 7) / 8; sz = (sz + 7UL) & ~7UL; - iommu->arena.map = kzalloc(sz, GFP_KERNEL); - if (!iommu->arena.map) { + iommu->tbl.map = kzalloc(sz, GFP_KERNEL); + if (!iommu->tbl.map) { printk(KERN_ERR PFX "Error, kmalloc(arena.map) failed.\n"); return -ENOMEM; } - iommu->arena.limit = num_tsb_entries; - - sz = probe_existing_entries(pbm, iommu); + iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT, + NULL, false /* no large_pool */, + 0 /* default npools */, + false /* want span boundary checking */); + sz = probe_existing_entries(pbm, &iommu->tbl); if (sz) printk("%s: Imported %lu TSB entries from OBP\n", pbm->name, sz); @@ -912,6 +1207,18 @@ static int pci_sun4v_pbm_init(struct pci_pbm_info *pbm, pci_sun4v_scan_bus(pbm, &op->dev); + /* if atu_init fails its not complete failure. + * we can still continue using legacy iommu. + */ + if (pbm->iommu->atu) { + err = pci_sun4v_atu_init(pbm); + if (err) { + kfree(pbm->iommu->atu); + pbm->iommu->atu = NULL; + pr_err(PFX "ATU init failed, err=%d\n", err); + } + } + pbm->next = pci_pbm_root; pci_pbm_root = pbm; @@ -925,23 +1232,41 @@ static int pci_sun4v_probe(struct platform_device *op) struct pci_pbm_info *pbm; struct device_node *dp; struct iommu *iommu; + struct atu *atu; u32 devhandle; - int i, err; + int i, err = -ENODEV; + static bool hv_atu = true; dp = op->dev.of_node; if (!hvapi_negotiated++) { - err = sun4v_hvapi_register(HV_GRP_PCI, - vpci_major, - &vpci_minor); + for (i = 0; i < ARRAY_SIZE(vpci_versions); i++) { + vpci_major = vpci_versions[i].major; + vpci_minor = vpci_versions[i].minor; + + err = sun4v_hvapi_register(HV_GRP_PCI, vpci_major, + &vpci_minor); + if (!err) + break; + } if (err) { - printk(KERN_ERR PFX "Could not register hvapi, " - "err=%d\n", err); + pr_err(PFX "Could not register hvapi, err=%d\n", err); return err; } - printk(KERN_INFO PFX "Registered hvapi major[%lu] minor[%lu]\n", - vpci_major, vpci_minor); + pr_info(PFX "Registered hvapi major[%lu] minor[%lu]\n", + vpci_major, vpci_minor); + + err = sun4v_hvapi_register(HV_GRP_ATU, vatu_major, &vatu_minor); + if (err) { + /* don't return an error if we fail to register the + * ATU group, but ATU hcalls won't be available. + */ + hv_atu = false; + } else { + pr_info(PFX "Registered hvapi ATU major[%lu] minor[%lu]\n", + vatu_major, vatu_minor); + } dma_ops = &sun4v_dma_ops; } @@ -980,6 +1305,14 @@ static int pci_sun4v_probe(struct platform_device *op) } pbm->iommu = iommu; + iommu->atu = NULL; + if (hv_atu) { + atu = kzalloc(sizeof(*atu), GFP_KERNEL); + if (!atu) + pr_err(PFX "Could not allocate atu\n"); + else + iommu->atu = atu; + } err = pci_sun4v_pbm_init(pbm, op, devhandle); if (err) @@ -990,6 +1323,7 @@ static int pci_sun4v_probe(struct platform_device *op) return 0; out_free_iommu: + kfree(iommu->atu); kfree(pbm->iommu); out_free_controller: @@ -1010,7 +1344,6 @@ static const struct of_device_id pci_sun4v_match[] = { static struct platform_driver pci_sun4v_driver = { .driver = { .name = DRIVER_NAME, - .owner = THIS_MODULE, .of_match_table = pci_sun4v_match, }, .probe = pci_sun4v_probe, diff --git a/arch/sparc/kernel/pci_sun4v.h b/arch/sparc/kernel/pci_sun4v.h index 8e9fc3a5b4f5..d47263a9901a 100644 --- a/arch/sparc/kernel/pci_sun4v.h +++ b/arch/sparc/kernel/pci_sun4v.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* pci_sun4v.h: SUN4V specific PCI controller support. * * Copyright (C) 2006 David S. Miller (davem@davemloft.net) @@ -6,87 +7,108 @@ #ifndef _PCI_SUN4V_H #define _PCI_SUN4V_H -extern long pci_sun4v_iommu_map(unsigned long devhandle, - unsigned long tsbid, - unsigned long num_ttes, - unsigned long io_attributes, - unsigned long io_page_list_pa); -extern unsigned long pci_sun4v_iommu_demap(unsigned long devhandle, - unsigned long tsbid, - unsigned long num_ttes); -extern unsigned long pci_sun4v_iommu_getmap(unsigned long devhandle, - unsigned long tsbid, - unsigned long *io_attributes, - unsigned long *real_address); -extern unsigned long pci_sun4v_config_get(unsigned long devhandle, - unsigned long pci_device, - unsigned long config_offset, - unsigned long size); -extern int pci_sun4v_config_put(unsigned long devhandle, - unsigned long pci_device, - unsigned long config_offset, - unsigned long size, - unsigned long data); +long pci_sun4v_iommu_map(unsigned long devhandle, + unsigned long tsbid, + unsigned long num_ttes, + unsigned long io_attributes, + unsigned long io_page_list_pa); +unsigned long pci_sun4v_iommu_demap(unsigned long devhandle, + unsigned long tsbid, + unsigned long num_ttes); +unsigned long pci_sun4v_iommu_getmap(unsigned long devhandle, + unsigned long tsbid, + unsigned long *io_attributes, + unsigned long *real_address); +unsigned long pci_sun4v_config_get(unsigned long devhandle, + unsigned long pci_device, + unsigned long config_offset, + unsigned long size); +int pci_sun4v_config_put(unsigned long devhandle, + unsigned long pci_device, + unsigned long config_offset, + unsigned long size, + unsigned long data); -extern unsigned long pci_sun4v_msiq_conf(unsigned long devhandle, +unsigned long pci_sun4v_msiq_conf(unsigned long devhandle, unsigned long msiqid, unsigned long msiq_paddr, unsigned long num_entries); -extern unsigned long pci_sun4v_msiq_info(unsigned long devhandle, - unsigned long msiqid, - unsigned long *msiq_paddr, - unsigned long *num_entries); -extern unsigned long pci_sun4v_msiq_getvalid(unsigned long devhandle, - unsigned long msiqid, - unsigned long *valid); -extern unsigned long pci_sun4v_msiq_setvalid(unsigned long devhandle, - unsigned long msiqid, - unsigned long valid); -extern unsigned long pci_sun4v_msiq_getstate(unsigned long devhandle, - unsigned long msiqid, - unsigned long *state); -extern unsigned long pci_sun4v_msiq_setstate(unsigned long devhandle, - unsigned long msiqid, - unsigned long state); -extern unsigned long pci_sun4v_msiq_gethead(unsigned long devhandle, - unsigned long msiqid, - unsigned long *head); -extern unsigned long pci_sun4v_msiq_sethead(unsigned long devhandle, - unsigned long msiqid, - unsigned long head); -extern unsigned long pci_sun4v_msiq_gettail(unsigned long devhandle, - unsigned long msiqid, - unsigned long *head); -extern unsigned long pci_sun4v_msi_getvalid(unsigned long devhandle, - unsigned long msinum, - unsigned long *valid); -extern unsigned long pci_sun4v_msi_setvalid(unsigned long devhandle, - unsigned long msinum, - unsigned long valid); -extern unsigned long pci_sun4v_msi_getmsiq(unsigned long devhandle, - unsigned long msinum, - unsigned long *msiq); -extern unsigned long pci_sun4v_msi_setmsiq(unsigned long devhandle, - unsigned long msinum, - unsigned long msiq, - unsigned long msitype); -extern unsigned long pci_sun4v_msi_getstate(unsigned long devhandle, - unsigned long msinum, - unsigned long *state); -extern unsigned long pci_sun4v_msi_setstate(unsigned long devhandle, - unsigned long msinum, - unsigned long state); -extern unsigned long pci_sun4v_msg_getmsiq(unsigned long devhandle, - unsigned long msinum, - unsigned long *msiq); -extern unsigned long pci_sun4v_msg_setmsiq(unsigned long devhandle, - unsigned long msinum, - unsigned long msiq); -extern unsigned long pci_sun4v_msg_getvalid(unsigned long devhandle, - unsigned long msinum, - unsigned long *valid); -extern unsigned long pci_sun4v_msg_setvalid(unsigned long devhandle, - unsigned long msinum, - unsigned long valid); +unsigned long pci_sun4v_msiq_info(unsigned long devhandle, + unsigned long msiqid, + unsigned long *msiq_paddr, + unsigned long *num_entries); +unsigned long pci_sun4v_msiq_getvalid(unsigned long devhandle, + unsigned long msiqid, + unsigned long *valid); +unsigned long pci_sun4v_msiq_setvalid(unsigned long devhandle, + unsigned long msiqid, + unsigned long valid); +unsigned long pci_sun4v_msiq_getstate(unsigned long devhandle, + unsigned long msiqid, + unsigned long *state); +unsigned long pci_sun4v_msiq_setstate(unsigned long devhandle, + unsigned long msiqid, + unsigned long state); +unsigned long pci_sun4v_msiq_gethead(unsigned long devhandle, + unsigned long msiqid, + unsigned long *head); +unsigned long pci_sun4v_msiq_sethead(unsigned long devhandle, + unsigned long msiqid, + unsigned long head); +unsigned long pci_sun4v_msiq_gettail(unsigned long devhandle, + unsigned long msiqid, + unsigned long *head); +unsigned long pci_sun4v_msi_getvalid(unsigned long devhandle, + unsigned long msinum, + unsigned long *valid); +unsigned long pci_sun4v_msi_setvalid(unsigned long devhandle, + unsigned long msinum, + unsigned long valid); +unsigned long pci_sun4v_msi_getmsiq(unsigned long devhandle, + unsigned long msinum, + unsigned long *msiq); +unsigned long pci_sun4v_msi_setmsiq(unsigned long devhandle, + unsigned long msinum, + unsigned long msiq, + unsigned long msitype); +unsigned long pci_sun4v_msi_getstate(unsigned long devhandle, + unsigned long msinum, + unsigned long *state); +unsigned long pci_sun4v_msi_setstate(unsigned long devhandle, + unsigned long msinum, + unsigned long state); +unsigned long pci_sun4v_msg_getmsiq(unsigned long devhandle, + unsigned long msinum, + unsigned long *msiq); +unsigned long pci_sun4v_msg_setmsiq(unsigned long devhandle, + unsigned long msinum, + unsigned long msiq); +unsigned long pci_sun4v_msg_getvalid(unsigned long devhandle, + unsigned long msinum, + unsigned long *valid); +unsigned long pci_sun4v_msg_setvalid(unsigned long devhandle, + unsigned long msinum, + unsigned long valid); +/* Sun4v HV IOMMU v2 APIs */ +unsigned long pci_sun4v_iotsb_conf(unsigned long devhandle, + unsigned long ra, + unsigned long table_size, + unsigned long page_size, + unsigned long dvma_base, + u64 *iotsb_num); +unsigned long pci_sun4v_iotsb_bind(unsigned long devhandle, + unsigned long iotsb_num, + unsigned int pci_device); +unsigned long pci_sun4v_iotsb_map(unsigned long devhandle, + unsigned long iotsb_num, + unsigned long iotsb_index_iottes, + unsigned long io_attributes, + unsigned long io_page_list_pa, + long *mapped); +unsigned long pci_sun4v_iotsb_demap(unsigned long devhandle, + unsigned long iotsb_num, + unsigned long iotsb_index, + unsigned long iottes, + unsigned long *demapped); #endif /* !(_PCI_SUN4V_H) */ diff --git a/arch/sparc/kernel/pci_sun4v_asm.S b/arch/sparc/kernel/pci_sun4v_asm.S index e606d46c6815..2b8051871a15 100644 --- a/arch/sparc/kernel/pci_sun4v_asm.S +++ b/arch/sparc/kernel/pci_sun4v_asm.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* pci_sun4v_asm: Hypervisor calls for PCI support. * * Copyright (C) 2006, 2008 David S. Miller <davem@davemloft.net> @@ -360,3 +361,71 @@ ENTRY(pci_sun4v_msg_setvalid) mov %o0, %o0 ENDPROC(pci_sun4v_msg_setvalid) + /* + * %o0: devhandle + * %o1: r_addr + * %o2: size + * %o3: pagesize + * %o4: virt + * %o5: &iotsb_num/&iotsb_handle + * + * returns %o0: status + * %o1: iotsb_num/iotsb_handle + */ +ENTRY(pci_sun4v_iotsb_conf) + mov %o5, %g1 + mov HV_FAST_PCI_IOTSB_CONF, %o5 + ta HV_FAST_TRAP + retl + stx %o1, [%g1] +ENDPROC(pci_sun4v_iotsb_conf) + + /* + * %o0: devhandle + * %o1: iotsb_num/iotsb_handle + * %o2: pci_device + * + * returns %o0: status + */ +ENTRY(pci_sun4v_iotsb_bind) + mov HV_FAST_PCI_IOTSB_BIND, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_sun4v_iotsb_bind) + + /* + * %o0: devhandle + * %o1: iotsb_num/iotsb_handle + * %o2: index_count + * %o3: iotte_attributes + * %o4: io_page_list_p + * %o5: &mapped + * + * returns %o0: status + * %o1: #mapped + */ +ENTRY(pci_sun4v_iotsb_map) + mov %o5, %g1 + mov HV_FAST_PCI_IOTSB_MAP, %o5 + ta HV_FAST_TRAP + retl + stx %o1, [%g1] +ENDPROC(pci_sun4v_iotsb_map) + + /* + * %o0: devhandle + * %o1: iotsb_num/iotsb_handle + * %o2: iotsb_index + * %o3: #iottes + * %o4: &demapped + * + * returns %o0: status + * %o1: #demapped + */ +ENTRY(pci_sun4v_iotsb_demap) + mov HV_FAST_PCI_IOTSB_DEMAP, %o5 + ta HV_FAST_TRAP + retl + stx %o1, [%o4] +ENDPROC(pci_sun4v_iotsb_demap) diff --git a/arch/sparc/kernel/pcic.c b/arch/sparc/kernel/pcic.c index 09f4fdd8d808..d7c911724435 100644 --- a/arch/sparc/kernel/pcic.c +++ b/arch/sparc/kernel/pcic.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * pcic.c: MicroSPARC-IIep PCI controller support * @@ -15,6 +16,7 @@ #include <linux/init.h> #include <linux/mm.h> #include <linux/slab.h> +#include <linux/string.h> #include <linux/jiffies.h> #include <asm/swift.h> /* for cache flushing. */ @@ -33,9 +35,10 @@ #include <asm/pcic.h> #include <asm/timex.h> #include <asm/timer.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/irq_regs.h> +#include "kernel.h" #include "irq.h" /* @@ -162,8 +165,8 @@ static int pcic0_up; static struct linux_pcic pcic0; void __iomem *pcic_regs; -volatile int pcic_speculative; -volatile int pcic_trapped; +static volatile int pcic_speculative; +static volatile int pcic_trapped; /* forward */ unsigned int pcic_build_device_irq(struct platform_device *op, @@ -329,7 +332,7 @@ int __init pcic_probe(void) pcic->pcic_res_cfg_addr.name = "pcic_cfg_addr"; if ((pcic->pcic_config_space_addr = - ioremap(regs[2].phys_addr, regs[2].reg_size * 2)) == 0) { + ioremap(regs[2].phys_addr, regs[2].reg_size * 2)) == NULL) { prom_printf("PCIC: Error, cannot map " "PCI Configuration Space Address.\n"); prom_halt(); @@ -341,7 +344,7 @@ int __init pcic_probe(void) */ pcic->pcic_res_cfg_data.name = "pcic_cfg_data"; if ((pcic->pcic_config_space_data = - ioremap(regs[3].phys_addr, regs[3].reg_size * 2)) == 0) { + ioremap(regs[3].phys_addr, regs[3].reg_size * 2)) == NULL) { prom_printf("PCIC: Error, cannot map " "PCI Configuration Space Data.\n"); prom_halt(); @@ -350,10 +353,9 @@ int __init pcic_probe(void) pbm = &pcic->pbm; pbm->prom_node = node; prom_getstring(node, "name", namebuf, 63); namebuf[63] = 0; - strcpy(pbm->prom_name, namebuf); + strscpy(pbm->prom_name, namebuf); { - extern volatile int t_nmi[4]; extern int pcic_nmi_trap_patch[4]; t_nmi[0] = pcic_nmi_trap_patch[0]; @@ -391,12 +393,16 @@ static void __init pcic_pbm_scan_bus(struct linux_pcic *pcic) struct linux_pbm_info *pbm = &pcic->pbm; pbm->pci_bus = pci_scan_bus(pbm->pci_first_busno, &pcic_ops, pbm); + if (!pbm->pci_bus) + return; + #if 0 /* deadwood transplanted from sparc64 */ pci_fill_in_pbm_cookies(pbm->pci_bus, pbm, pbm->prom_node); pci_record_assignments(pbm, pbm->pci_bus); pci_assign_unassigned(pbm, pbm->pci_bus); pci_fixup_irq(pbm, pbm->pci_bus); #endif + pci_bus_add_devices(pbm->pci_bus); } /* @@ -472,7 +478,7 @@ static void pcic_map_pci_device(struct linux_pcic *pcic, int j; if (node == 0 || node == -1) { - strcpy(namebuf, "???"); + strscpy(namebuf, "???"); } else { prom_getstring(node, "name", namebuf, 63); namebuf[63] = 0; } @@ -513,10 +519,10 @@ static void pcic_map_pci_device(struct linux_pcic *pcic, * board in a PCI slot. We must remap it * under 64K but it is not done yet. XXX */ - printk("PCIC: Skipping I/O space at 0x%lx, " - "this will Oops if a driver attaches " - "device '%s' at %02x:%02x)\n", address, - namebuf, dev->bus->number, dev->devfn); + pci_info(dev, "PCIC: Skipping I/O space at " + "0x%lx, this will Oops if a driver " + "attaches device '%s'\n", address, + namebuf); } } } @@ -531,12 +537,12 @@ pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node) char namebuf[64]; if (node == 0 || node == -1) { - strcpy(namebuf, "???"); + strscpy(namebuf, "???"); } else { prom_getstring(node, "name", namebuf, sizeof(namebuf)); } - if ((p = pcic->pcic_imap) == 0) { + if ((p = pcic->pcic_imap) == NULL) { dev->irq = 0; return; } @@ -546,8 +552,8 @@ pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node) p++; } if (i >= pcic->pcic_imdim) { - printk("PCIC: device %s devfn %02x:%02x not found in %d\n", - namebuf, dev->bus->number, dev->devfn, pcic->pcic_imdim); + pci_info(dev, "PCIC: device %s not found in %d\n", namebuf, + pcic->pcic_imdim); dev->irq = 0; return; } @@ -560,7 +566,7 @@ pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node) ivec = readw(pcic->pcic_regs+PCI_INT_SELECT_HI); real_irq = ivec >> ((i-4) << 2) & 0xF; } else { /* Corrupted map */ - printk("PCIC: BAD PIN %d\n", i); for (;;) {} + pci_info(dev, "PCIC: BAD PIN %d\n", i); for (;;) {} } /* P3 */ /* printk("PCIC: device %s pin %d ivec 0x%x irq %x\n", namebuf, i, ivec, dev->irq); */ @@ -569,10 +575,10 @@ pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node) */ if (real_irq == 0 || p->force) { if (p->irq == 0 || p->irq >= 15) { /* Corrupted map */ - printk("PCIC: BAD IRQ %d\n", p->irq); for (;;) {} + pci_info(dev, "PCIC: BAD IRQ %d\n", p->irq); for (;;) {} } - printk("PCIC: setting irq %d at pin %d for device %02x:%02x\n", - p->irq, p->pin, dev->bus->number, dev->devfn); + pci_info(dev, "PCIC: setting irq %d at pin %d\n", p->irq, + p->pin); real_irq = p->irq; i = p->pin; @@ -597,15 +603,13 @@ pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node) void pcibios_fixup_bus(struct pci_bus *bus) { struct pci_dev *dev; - int i, has_io, has_mem; - unsigned int cmd; struct linux_pcic *pcic; /* struct linux_pbm_info* pbm = &pcic->pbm; */ int node; struct pcidev_cookie *pcp; if (!pcic0_up) { - printk("pcibios_fixup_bus: no PCIC\n"); + pci_info(bus, "pcibios_fixup_bus: no PCIC\n"); return; } pcic = &pcic0; @@ -614,44 +618,12 @@ void pcibios_fixup_bus(struct pci_bus *bus) * Next crud is an equivalent of pbm = pcic_bus_to_pbm(bus); */ if (bus->number != 0) { - printk("pcibios_fixup_bus: nonzero bus 0x%x\n", bus->number); + pci_info(bus, "pcibios_fixup_bus: nonzero bus 0x%x\n", + bus->number); return; } list_for_each_entry(dev, &bus->devices, bus_list) { - - /* - * Comment from i386 branch: - * There are buggy BIOSes that forget to enable I/O and memory - * access to PCI devices. We try to fix this, but we need to - * be sure that the BIOS didn't forget to assign an address - * to the device. [mj] - * OBP is a case of such BIOS :-) - */ - has_io = has_mem = 0; - for(i=0; i<6; i++) { - unsigned long f = dev->resource[i].flags; - if (f & IORESOURCE_IO) { - has_io = 1; - } else if (f & IORESOURCE_MEM) - has_mem = 1; - } - pcic_read_config(dev->bus, dev->devfn, PCI_COMMAND, 2, &cmd); - if (has_io && !(cmd & PCI_COMMAND_IO)) { - printk("PCIC: Enabling I/O for device %02x:%02x\n", - dev->bus->number, dev->devfn); - cmd |= PCI_COMMAND_IO; - pcic_write_config(dev->bus, dev->devfn, - PCI_COMMAND, 2, cmd); - } - if (has_mem && !(cmd & PCI_COMMAND_MEMORY)) { - printk("PCIC: Enabling memory for device %02x:%02x\n", - dev->bus->number, dev->devfn); - cmd |= PCI_COMMAND_MEMORY; - pcic_write_config(dev->bus, dev->devfn, - PCI_COMMAND, 2, cmd); - } - node = pdev_to_pnode(&pcic->pbm, dev); if(node == 0) node = -1; @@ -670,30 +642,6 @@ void pcibios_fixup_bus(struct pci_bus *bus) } } -/* - * pcic_pin_to_irq() is exported to bus probing code - */ -unsigned int -pcic_pin_to_irq(unsigned int pin, const char *name) -{ - struct linux_pcic *pcic = &pcic0; - unsigned int irq; - unsigned int ivec; - - if (pin < 4) { - ivec = readw(pcic->pcic_regs+PCI_INT_SELECT_LO); - irq = ivec >> (pin << 2) & 0xF; - } else if (pin < 8) { - ivec = readw(pcic->pcic_regs+PCI_INT_SELECT_HI); - irq = ivec >> ((pin-4) << 2) & 0xF; - } else { /* Corrupted map */ - printk("PCIC: BAD PIN %d FOR %s\n", pin, name); - for (;;) {} /* XXX Cannot panic properly in case of PROLL */ - } -/* P3 */ /* printk("PCIC: dev %s pin %d ivec 0x%x irq %x\n", name, pin, ivec, irq); */ - return irq; -} - /* Makes compiler happy */ static volatile int pcic_timer_dummy; @@ -766,24 +714,12 @@ static void watchdog_reset() { } #endif -resource_size_t pcibios_align_resource(void *data, const struct resource *res, - resource_size_t size, resource_size_t align) -{ - return res->start; -} - -int pcibios_enable_device(struct pci_dev *pdev, int mask) -{ - return 0; -} - /* * NMI */ void pcic_nmi(unsigned int pend, struct pt_regs *regs) { - - pend = flip_dword(pend); + pend = swab32(pend); if (!pcic_speculative || (pend & PCI_SYS_INT_PENDING_PIO) == 0) { /* @@ -875,82 +811,4 @@ void __init sun4m_pci_init_IRQ(void) sparc_config.load_profile_irq = pcic_load_profile_irq; } -/* - * This probably belongs here rather than ioport.c because - * we do not want this crud linked into SBus kernels. - * Also, think for a moment about likes of floppy.c that - * include architecture specific parts. They may want to redefine ins/outs. - * - * We do not use horrible macros here because we want to - * advance pointer by sizeof(size). - */ -void outsb(unsigned long addr, const void *src, unsigned long count) -{ - while (count) { - count -= 1; - outb(*(const char *)src, addr); - src += 1; - /* addr += 1; */ - } -} -EXPORT_SYMBOL(outsb); - -void outsw(unsigned long addr, const void *src, unsigned long count) -{ - while (count) { - count -= 2; - outw(*(const short *)src, addr); - src += 2; - /* addr += 2; */ - } -} -EXPORT_SYMBOL(outsw); - -void outsl(unsigned long addr, const void *src, unsigned long count) -{ - while (count) { - count -= 4; - outl(*(const long *)src, addr); - src += 4; - /* addr += 4; */ - } -} -EXPORT_SYMBOL(outsl); - -void insb(unsigned long addr, void *dst, unsigned long count) -{ - while (count) { - count -= 1; - *(unsigned char *)dst = inb(addr); - dst += 1; - /* addr += 1; */ - } -} -EXPORT_SYMBOL(insb); - -void insw(unsigned long addr, void *dst, unsigned long count) -{ - while (count) { - count -= 2; - *(unsigned short *)dst = inw(addr); - dst += 2; - /* addr += 2; */ - } -} -EXPORT_SYMBOL(insw); - -void insl(unsigned long addr, void *dst, unsigned long count) -{ - while (count) { - count -= 4; - /* - * XXX I am sure we are in for an unaligned trap here. - */ - *(unsigned long *)dst = inl(addr); - dst += 4; - /* addr += 4; */ - } -} -EXPORT_SYMBOL(insl); - subsys_initcall(pcic_init); diff --git a/arch/sparc/kernel/pcr.c b/arch/sparc/kernel/pcr.c index 269af58497aa..2a12c86af956 100644 --- a/arch/sparc/kernel/pcr.c +++ b/arch/sparc/kernel/pcr.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* pcr.c: Generic sparc64 performance counter infrastructure. * * Copyright (C) 2009 David S. Miller (davem@davemloft.net) @@ -191,12 +192,66 @@ static const struct pcr_ops n4_pcr_ops = { .pcr_nmi_disable = PCR_N4_PICNPT, }; +static u64 n5_pcr_read(unsigned long reg_num) +{ + unsigned long val; + + (void) sun4v_t5_get_perfreg(reg_num, &val); + + return val; +} + +static void n5_pcr_write(unsigned long reg_num, u64 val) +{ + (void) sun4v_t5_set_perfreg(reg_num, val); +} + +static const struct pcr_ops n5_pcr_ops = { + .read_pcr = n5_pcr_read, + .write_pcr = n5_pcr_write, + .read_pic = n4_pic_read, + .write_pic = n4_pic_write, + .nmi_picl_value = n4_picl_value, + .pcr_nmi_enable = (PCR_N4_PICNPT | PCR_N4_STRACE | + PCR_N4_UTRACE | PCR_N4_TOE | + (26 << PCR_N4_SL_SHIFT)), + .pcr_nmi_disable = PCR_N4_PICNPT, +}; + +static u64 m7_pcr_read(unsigned long reg_num) +{ + unsigned long val; + + (void) sun4v_m7_get_perfreg(reg_num, &val); + + return val; +} + +static void m7_pcr_write(unsigned long reg_num, u64 val) +{ + (void) sun4v_m7_set_perfreg(reg_num, val); +} + +static const struct pcr_ops m7_pcr_ops = { + .read_pcr = m7_pcr_read, + .write_pcr = m7_pcr_write, + .read_pic = n4_pic_read, + .write_pic = n4_pic_write, + .nmi_picl_value = n4_picl_value, + .pcr_nmi_enable = (PCR_N4_PICNPT | PCR_N4_STRACE | + PCR_N4_UTRACE | PCR_N4_TOE | + (26 << PCR_N4_SL_SHIFT)), + .pcr_nmi_disable = PCR_N4_PICNPT, +}; + static unsigned long perf_hsvc_group; static unsigned long perf_hsvc_major; static unsigned long perf_hsvc_minor; static int __init register_perf_hsvc(void) { + unsigned long hverror; + if (tlb_type == hypervisor) { switch (sun4v_chip_type) { case SUN4V_CHIP_NIAGARA1: @@ -215,6 +270,14 @@ static int __init register_perf_hsvc(void) perf_hsvc_group = HV_GRP_VT_CPU; break; + case SUN4V_CHIP_NIAGARA5: + perf_hsvc_group = HV_GRP_T5_CPU; + break; + + case SUN4V_CHIP_SPARC_M7: + perf_hsvc_group = HV_GRP_M7_PERF; + break; + default: return -ENODEV; } @@ -222,10 +285,12 @@ static int __init register_perf_hsvc(void) perf_hsvc_major = 1; perf_hsvc_minor = 0; - if (sun4v_hvapi_register(perf_hsvc_group, - perf_hsvc_major, - &perf_hsvc_minor)) { - printk("perfmon: Could not register hvapi.\n"); + hverror = sun4v_hvapi_register(perf_hsvc_group, + perf_hsvc_major, + &perf_hsvc_minor); + if (hverror) { + pr_err("perfmon: Could not register hvapi(0x%lx).\n", + hverror); return -ENODEV; } } @@ -254,6 +319,14 @@ static int __init setup_sun4v_pcr_ops(void) pcr_ops = &n4_pcr_ops; break; + case SUN4V_CHIP_NIAGARA5: + pcr_ops = &n5_pcr_ops; + break; + + case SUN4V_CHIP_SPARC_M7: + pcr_ops = &m7_pcr_ops; + break; + default: ret = -ENODEV; break; @@ -286,7 +359,7 @@ int __init pcr_arch_init(void) * counter overflow interrupt so we can't make use of * their hardware currently. */ - /* fallthrough */ + fallthrough; default: err = -ENODEV; goto out_unregister; diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c index b5c38faa4ead..cae4d33002a5 100644 --- a/arch/sparc/kernel/perf_event.c +++ b/arch/sparc/kernel/perf_event.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* Performance event support for sparc64. * * Copyright (C) 2009, 2010 David S. Miller <davem@davemloft.net> @@ -9,7 +10,7 @@ * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar * Copyright (C) 2009 Jaswinder Singh Rajput * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter - * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> + * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra */ #include <linux/perf_event.h> @@ -21,8 +22,9 @@ #include <asm/stacktrace.h> #include <asm/cpudata.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <linux/atomic.h> +#include <linux/sched/clock.h> #include <asm/nmi.h> #include <asm/pcr.h> #include <asm/cacheflush.h> @@ -108,9 +110,9 @@ struct cpu_hw_events { /* Enabled/disable state. */ int enabled; - unsigned int group_flag; + unsigned int txn_flags; }; -DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, }; +static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, }; /* An event map describes the characteristics of a performance * counter event. In particular it gives the encoding as well as @@ -737,25 +739,9 @@ static void sparc_vt_write_pmc(int idx, u64 val) { u64 pcr; - /* There seems to be an internal latch on the overflow event - * on SPARC-T4 that prevents it from triggering unless you - * update the PIC exactly as we do here. The requirement - * seems to be that you have to turn off event counting in the - * PCR around the PIC update. - * - * For example, after the following sequence: - * - * 1) set PIC to -1 - * 2) enable event counting and overflow reporting in PCR - * 3) overflow triggers, softint 15 handler invoked - * 4) clear OV bit in PCR - * 5) write PIC to -1 - * - * a subsequent overflow event will not trigger. This - * sequence works on SPARC-T3 and previous chips. - */ pcr = pcr_ops->read_pcr(idx); - pcr_ops->write_pcr(idx, PCR_N4_PICNPT); + /* ensure ov and ntc are reset */ + pcr &= ~(PCR_N4_OV | PCR_N4_NTC); pcr_ops->write_pic(idx, val & 0xffffffff); @@ -792,6 +778,29 @@ static const struct sparc_pmu niagara4_pmu = { .num_pic_regs = 4, }; +static const struct sparc_pmu sparc_m7_pmu = { + .event_map = niagara4_event_map, + .cache_map = &niagara4_cache_map, + .max_events = ARRAY_SIZE(niagara4_perfmon_event_map), + .read_pmc = sparc_vt_read_pmc, + .write_pmc = sparc_vt_write_pmc, + .upper_shift = 5, + .lower_shift = 5, + .event_mask = 0x7ff, + .user_bit = PCR_N4_UTRACE, + .priv_bit = PCR_N4_STRACE, + + /* We explicitly don't support hypervisor tracing. */ + .hv_bit = 0, + + .irq_bit = PCR_N4_TOE, + .upper_nop = 0, + .lower_nop = 0, + .flags = 0, + .max_hw_events = 4, + .num_pcrs = 4, + .num_pic_regs = 4, +}; static const struct sparc_pmu *sparc_pmu __read_mostly; static u64 event_encoding(u64 event_id, int idx) @@ -882,6 +891,10 @@ static int sparc_perf_event_set_period(struct perf_event *event, s64 period = hwc->sample_period; int ret = 0; + /* The period may have been changed by PERF_EVENT_IOC_PERIOD */ + if (unlikely(period != hwc->last_period)) + left = period - (hwc->last_period - left); + if (unlikely(left <= -period)) { left = period; local64_set(&hwc->period_left, left); @@ -919,6 +932,8 @@ static void read_in_all_counters(struct cpu_hw_events *cpuc) sparc_perf_event_update(cp, &cp->hw, cpuc->current_idx[i]); cpuc->current_idx[i] = PIC_NO_INDEX; + if (cp->hw.state & PERF_HES_STOPPED) + cp->hw.state |= PERF_HES_ARCH; } } } @@ -951,16 +966,20 @@ static void calculate_single_pcr(struct cpu_hw_events *cpuc) enc = perf_event_get_enc(cpuc->events[i]); cpuc->pcr[0] &= ~mask_for_index(idx); - if (hwc->state & PERF_HES_STOPPED) + if (hwc->state & PERF_HES_ARCH) { cpuc->pcr[0] |= nop_for_index(idx); - else + } else { cpuc->pcr[0] |= event_encoding(enc, idx); + hwc->state = 0; + } } out: cpuc->pcr[0] |= cpuc->event[0]->hw.config_base; } -/* On this PMU each PIC has it's own PCR control register. */ +static void sparc_pmu_start(struct perf_event *event, int flags); + +/* On this PMU each PIC has its own PCR control register. */ static void calculate_multiple_pcrs(struct cpu_hw_events *cpuc) { int i; @@ -972,20 +991,16 @@ static void calculate_multiple_pcrs(struct cpu_hw_events *cpuc) struct perf_event *cp = cpuc->event[i]; struct hw_perf_event *hwc = &cp->hw; int idx = hwc->idx; - u64 enc; if (cpuc->current_idx[i] != PIC_NO_INDEX) continue; - sparc_perf_event_set_period(cp, hwc, idx); cpuc->current_idx[i] = idx; - enc = perf_event_get_enc(cpuc->events[i]); - cpuc->pcr[idx] &= ~mask_for_index(idx); - if (hwc->state & PERF_HES_STOPPED) - cpuc->pcr[idx] |= nop_for_index(idx); - else - cpuc->pcr[idx] |= event_encoding(enc, idx); + if (cp->hw.state & PERF_HES_ARCH) + continue; + + sparc_pmu_start(cp, PERF_EF_RELOAD); } out: for (i = 0; i < cpuc->n_events; i++) { @@ -1013,7 +1028,7 @@ static void update_pcrs_for_enable(struct cpu_hw_events *cpuc) static void sparc_pmu_enable(struct pmu *pmu) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int i; if (cpuc->enabled) @@ -1031,7 +1046,7 @@ static void sparc_pmu_enable(struct pmu *pmu) static void sparc_pmu_disable(struct pmu *pmu) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int i; if (!cpuc->enabled) @@ -1065,7 +1080,7 @@ static int active_event_index(struct cpu_hw_events *cpuc, static void sparc_pmu_start(struct perf_event *event, int flags) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int idx = active_event_index(cpuc, event); if (flags & PERF_EF_RELOAD) { @@ -1076,11 +1091,13 @@ static void sparc_pmu_start(struct perf_event *event, int flags) event->hw.state = 0; sparc_pmu_enable_event(cpuc, &event->hw, idx); + + perf_event_update_userpage(event); } static void sparc_pmu_stop(struct perf_event *event, int flags) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int idx = active_event_index(cpuc, event); if (!(event->hw.state & PERF_HES_STOPPED)) { @@ -1096,12 +1113,11 @@ static void sparc_pmu_stop(struct perf_event *event, int flags) static void sparc_pmu_del(struct perf_event *event, int _flags) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); unsigned long flags; int i; local_irq_save(flags); - perf_pmu_disable(event->pmu); for (i = 0; i < cpuc->n_events; i++) { if (event == cpuc->event[i]) { @@ -1127,13 +1143,12 @@ static void sparc_pmu_del(struct perf_event *event, int _flags) } } - perf_pmu_enable(event->pmu); local_irq_restore(flags); } static void sparc_pmu_read(struct perf_event *event) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int idx = active_event_index(cpuc, event); struct hw_perf_event *hwc = &event->hw; @@ -1145,7 +1160,7 @@ static DEFINE_MUTEX(pmc_grab_mutex); static void perf_stop_nmi_watchdog(void *unused) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int i; stop_nmi_watchdog(NULL); @@ -1153,7 +1168,7 @@ static void perf_stop_nmi_watchdog(void *unused) cpuc->pcr[i] = pcr_ops->read_pcr(i); } -void perf_event_grab_pmc(void) +static void perf_event_grab_pmc(void) { if (atomic_inc_not_zero(&active_events)) return; @@ -1169,7 +1184,7 @@ void perf_event_grab_pmc(void) mutex_unlock(&pmc_grab_mutex); } -void perf_event_release_pmc(void) +static void perf_event_release_pmc(void) { if (atomic_dec_and_mutex_lock(&active_events, &pmc_grab_mutex)) { if (atomic_read(&nmi_active) == 0) @@ -1341,7 +1356,7 @@ static int collect_events(struct perf_event *group, int max_count, events[n] = group->hw.event_base; current_idx[n++] = PIC_NO_INDEX; } - list_for_each_entry(event, &group->sibling_list, group_entry) { + for_each_sibling_event(event, group) { if (!is_software_event(event) && event->state != PERF_EVENT_STATE_OFF) { if (n >= max_count) @@ -1356,12 +1371,11 @@ static int collect_events(struct perf_event *group, int max_count, static int sparc_pmu_add(struct perf_event *event, int ef_flags) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int n0, ret = -EAGAIN; unsigned long flags; local_irq_save(flags); - perf_pmu_disable(event->pmu); n0 = cpuc->n_events; if (n0 >= sparc_pmu->max_hw_events) @@ -1371,16 +1385,16 @@ static int sparc_pmu_add(struct perf_event *event, int ef_flags) cpuc->events[n0] = event->hw.event_base; cpuc->current_idx[n0] = PIC_NO_INDEX; - event->hw.state = PERF_HES_UPTODATE; + event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; if (!(ef_flags & PERF_EF_START)) - event->hw.state |= PERF_HES_STOPPED; + event->hw.state |= PERF_HES_ARCH; /* * If group events scheduling transaction was started, * skip the schedulability test here, it will be performed * at commit time(->commit_txn) as a whole */ - if (cpuc->group_flag & PERF_EVENT_TXN) + if (cpuc->txn_flags & PERF_PMU_TXN_ADD) goto nocheck; if (check_excludes(cpuc->event, n0, 1)) @@ -1394,7 +1408,6 @@ nocheck: ret = 0; out: - perf_pmu_enable(event->pmu); local_irq_restore(flags); return ret; } @@ -1496,12 +1509,17 @@ static int sparc_pmu_event_init(struct perf_event *event) * Set the flag to make pmu::enable() not perform the * schedulability test, it will be performed at commit time */ -static void sparc_pmu_start_txn(struct pmu *pmu) +static void sparc_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) { - struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); + + WARN_ON_ONCE(cpuhw->txn_flags); /* txn already in flight */ + + cpuhw->txn_flags = txn_flags; + if (txn_flags & ~PERF_PMU_TXN_ADD) + return; perf_pmu_disable(pmu); - cpuhw->group_flag |= PERF_EVENT_TXN; } /* @@ -1511,9 +1529,16 @@ static void sparc_pmu_start_txn(struct pmu *pmu) */ static void sparc_pmu_cancel_txn(struct pmu *pmu) { - struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); + unsigned int txn_flags; + + WARN_ON_ONCE(!cpuhw->txn_flags); /* no txn in flight */ + + txn_flags = cpuhw->txn_flags; + cpuhw->txn_flags = 0; + if (txn_flags & ~PERF_PMU_TXN_ADD) + return; - cpuhw->group_flag &= ~PERF_EVENT_TXN; perf_pmu_enable(pmu); } @@ -1524,20 +1549,26 @@ static void sparc_pmu_cancel_txn(struct pmu *pmu) */ static int sparc_pmu_commit_txn(struct pmu *pmu) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int n; if (!sparc_pmu) return -EINVAL; - cpuc = &__get_cpu_var(cpu_hw_events); + WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ + + if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { + cpuc->txn_flags = 0; + return 0; + } + n = cpuc->n_events; if (check_excludes(cpuc->event, 0, n)) return -EINVAL; if (sparc_check_constraints(cpuc->event, cpuc->events, n)) return -EAGAIN; - cpuc->group_flag &= ~PERF_EVENT_TXN; + cpuc->txn_flags = 0; perf_pmu_enable(pmu); return 0; } @@ -1586,6 +1617,8 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, struct perf_sample_data data; struct cpu_hw_events *cpuc; struct pt_regs *regs; + u64 finish_clock; + u64 start_clock; int i; if (!atomic_read(&active_events)) @@ -1599,9 +1632,11 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, return NOTIFY_DONE; } + start_clock = sched_clock(); + regs = args->regs; - cpuc = &__get_cpu_var(cpu_hw_events); + cpuc = this_cpu_ptr(&cpu_hw_events); /* If the PMU has the TOE IRQ enable bits, we need to do a * dummy write to the %pcr to clear the overflow bits and thus @@ -1633,10 +1668,13 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, if (!sparc_perf_event_set_period(event, hwc, idx)) continue; - if (perf_event_overflow(event, &data, regs)) - sparc_pmu_stop(event, 0); + perf_event_overflow(event, &data, regs); } + finish_clock = sched_clock(); + + perf_sample_event_took(finish_clock - start_clock); + return NOTIFY_STOP; } @@ -1662,18 +1700,26 @@ static bool __init supported_pmu(void) sparc_pmu = &niagara2_pmu; return true; } - if (!strcmp(sparc_pmu_type, "niagara4")) { + if (!strcmp(sparc_pmu_type, "niagara4") || + !strcmp(sparc_pmu_type, "niagara5")) { sparc_pmu = &niagara4_pmu; return true; } + if (!strcmp(sparc_pmu_type, "sparc-m7")) { + sparc_pmu = &sparc_m7_pmu; + return true; + } return false; } -int __init init_hw_perf_events(void) +static int __init init_hw_perf_events(void) { + int err; + pr_info("Performance events: "); - if (!supported_pmu()) { + err = pcr_arch_init(); + if (err || !supported_pmu()) { pr_cont("No support for PMU type '%s'\n", sparc_pmu_type); return 0; } @@ -1685,9 +1731,9 @@ int __init init_hw_perf_events(void) return 0; } -early_initcall(init_hw_perf_events); +pure_initcall(init_hw_perf_events); -void perf_callchain_kernel(struct perf_callchain_entry *entry, +void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { unsigned long ksp, fp; @@ -1724,78 +1770,107 @@ void perf_callchain_kernel(struct perf_callchain_entry *entry, perf_callchain_store(entry, pc); #ifdef CONFIG_FUNCTION_GRAPH_TRACER if ((pc + 8UL) == (unsigned long) &return_to_handler) { - int index = current->curr_ret_stack; - if (current->ret_stack && index >= graph) { - pc = current->ret_stack[index - graph].ret; + struct ftrace_ret_stack *ret_stack; + ret_stack = ftrace_graph_get_ret_stack(current, + graph); + if (ret_stack) { + pc = ret_stack->ret; perf_callchain_store(entry, pc); graph++; } } #endif - } while (entry->nr < PERF_MAX_STACK_DEPTH); + } while (entry->nr < entry->max_stack); +} + +static inline int +valid_user_frame(const void __user *fp, unsigned long size) +{ + /* addresses should be at least 4-byte aligned */ + if (((unsigned long) fp) & 3) + return 0; + + return (__range_not_ok(fp, size, TASK_SIZE) == 0); } -static void perf_callchain_user_64(struct perf_callchain_entry *entry, +static void perf_callchain_user_64(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { unsigned long ufp; - ufp = regs->u_regs[UREG_I6] + STACK_BIAS; + ufp = regs->u_regs[UREG_FP] + STACK_BIAS; do { - struct sparc_stackf *usf, sf; + struct sparc_stackf __user *usf; + struct sparc_stackf sf; unsigned long pc; - usf = (struct sparc_stackf *) ufp; + usf = (struct sparc_stackf __user *)ufp; + if (!valid_user_frame(usf, sizeof(sf))) + break; + if (__copy_from_user_inatomic(&sf, usf, sizeof(sf))) break; pc = sf.callers_pc; ufp = (unsigned long)sf.fp + STACK_BIAS; perf_callchain_store(entry, pc); - } while (entry->nr < PERF_MAX_STACK_DEPTH); + } while (entry->nr < entry->max_stack); } -static void perf_callchain_user_32(struct perf_callchain_entry *entry, +static void perf_callchain_user_32(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { unsigned long ufp; - ufp = regs->u_regs[UREG_I6] & 0xffffffffUL; + ufp = regs->u_regs[UREG_FP] & 0xffffffffUL; do { unsigned long pc; if (thread32_stack_is_64bit(ufp)) { - struct sparc_stackf *usf, sf; + struct sparc_stackf __user *usf; + struct sparc_stackf sf; ufp += STACK_BIAS; - usf = (struct sparc_stackf *) ufp; + usf = (struct sparc_stackf __user *)ufp; if (__copy_from_user_inatomic(&sf, usf, sizeof(sf))) break; pc = sf.callers_pc & 0xffffffff; ufp = ((unsigned long) sf.fp) & 0xffffffff; } else { - struct sparc_stackf32 *usf, sf; - usf = (struct sparc_stackf32 *) ufp; + struct sparc_stackf32 __user *usf; + struct sparc_stackf32 sf; + usf = (struct sparc_stackf32 __user *)ufp; if (__copy_from_user_inatomic(&sf, usf, sizeof(sf))) break; pc = sf.callers_pc; ufp = (unsigned long)sf.fp; } perf_callchain_store(entry, pc); - } while (entry->nr < PERF_MAX_STACK_DEPTH); + } while (entry->nr < entry->max_stack); } void -perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs) +perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { + u64 saved_fault_address = current_thread_info()->fault_address; + u8 saved_fault_code = get_thread_fault_code(); + perf_callchain_store(entry, regs->tpc); if (!current->mm) return; flushw_user(); + + pagefault_disable(); + if (test_thread_flag(TIF_32BIT)) perf_callchain_user_32(entry, regs); else perf_callchain_user_64(entry, regs); + + pagefault_enable(); + + set_thread_fault_code(saved_fault_code); + current_thread_info()->fault_address = saved_fault_address; } diff --git a/arch/sparc/kernel/pmc.c b/arch/sparc/kernel/pmc.c index 8b7297faca79..69a0206e56f0 100644 --- a/arch/sparc/kernel/pmc.c +++ b/arch/sparc/kernel/pmc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* pmc - Driver implementation for power management functions * of Power Management Controller (PMC) on SPARCstation-Voyager. * @@ -10,12 +11,12 @@ #include <linux/init.h> #include <linux/pm.h> #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/platform_device.h> #include <linux/module.h> #include <asm/io.h> #include <asm/oplib.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/auxio.h> #include <asm/processor.h> @@ -71,7 +72,7 @@ static int pmc_probe(struct platform_device *op) return 0; } -static struct of_device_id pmc_match[] = { +static const struct of_device_id pmc_match[] = { { .name = PMC_OBPNAME, }, @@ -82,7 +83,6 @@ MODULE_DEVICE_TABLE(of, pmc_match); static struct platform_driver pmc_driver = { .driver = { .name = "pmc", - .owner = THIS_MODULE, .of_match_table = pmc_match, }, .probe = pmc_probe, diff --git a/arch/sparc/kernel/power.c b/arch/sparc/kernel/power.c index 4cb23c41553f..db8a3f9e3d40 100644 --- a/arch/sparc/kernel/power.c +++ b/arch/sparc/kernel/power.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* power.c: Power management driver. * * Copyright (C) 1999, 2007, 2008 David S. Miller (davem@davemloft.net) @@ -8,7 +9,8 @@ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/reboot.h> -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/platform_device.h> #include <asm/prom.h> #include <asm/io.h> @@ -27,7 +29,7 @@ static int has_button_interrupt(unsigned int irq, struct device_node *dp) { if (irq == 0xffffffff) return 0; - if (!of_find_property(dp, "button", NULL)) + if (!of_property_read_bool(dp, "button")) return 0; return 1; @@ -40,8 +42,8 @@ static int power_probe(struct platform_device *op) power_reg = of_ioremap(res, 0, 0x4, "power"); - printk(KERN_INFO "%s: Control reg at %llx\n", - op->dev.of_node->name, res->start); + printk(KERN_INFO "%pOFn: Control reg at %llx\n", + op->dev.of_node, res->start); if (has_button_interrupt(irq, op->dev.of_node)) { if (request_irq(irq, @@ -63,14 +65,8 @@ static struct platform_driver power_driver = { .probe = power_probe, .driver = { .name = "power", - .owner = THIS_MODULE, .of_match_table = power_match, }, }; -static int __init power_init(void) -{ - return platform_driver_register(&power_driver); -} - -device_initcall(power_init); +builtin_platform_driver(power_driver); diff --git a/arch/sparc/kernel/process.c b/arch/sparc/kernel/process.c new file mode 100644 index 000000000000..0442ab00518d --- /dev/null +++ b/arch/sparc/kernel/process.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * This file handles the architecture independent parts of process handling.. + */ + +#include <linux/compat.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/ptrace.h> +#include <linux/sched.h> +#include <linux/sched/task.h> +#include <linux/sched/task_stack.h> +#include <linux/signal.h> + +#include "kernel.h" + +asmlinkage long sparc_fork(struct pt_regs *regs) +{ + unsigned long orig_i1 = regs->u_regs[UREG_I1]; + long ret; + struct kernel_clone_args args = { + .exit_signal = SIGCHLD, + /* Reuse the parent's stack for the child. */ + .stack = regs->u_regs[UREG_FP], + }; + + ret = kernel_clone(&args); + + /* If we get an error and potentially restart the system + * call, we're screwed because copy_thread() clobbered + * the parent's %o1. So detect that case and restore it + * here. + */ + if ((unsigned long)ret >= -ERESTART_RESTARTBLOCK) + regs->u_regs[UREG_I1] = orig_i1; + + return ret; +} + +asmlinkage long sparc_vfork(struct pt_regs *regs) +{ + unsigned long orig_i1 = regs->u_regs[UREG_I1]; + long ret; + + struct kernel_clone_args args = { + .flags = CLONE_VFORK | CLONE_VM, + .exit_signal = SIGCHLD, + /* Reuse the parent's stack for the child. */ + .stack = regs->u_regs[UREG_FP], + }; + + ret = kernel_clone(&args); + + /* If we get an error and potentially restart the system + * call, we're screwed because copy_thread() clobbered + * the parent's %o1. So detect that case and restore it + * here. + */ + if ((unsigned long)ret >= -ERESTART_RESTARTBLOCK) + regs->u_regs[UREG_I1] = orig_i1; + + return ret; +} + +asmlinkage long sparc_clone(struct pt_regs *regs) +{ + unsigned long orig_i1 = regs->u_regs[UREG_I1]; + unsigned int flags = lower_32_bits(regs->u_regs[UREG_I0]); + long ret; + + struct kernel_clone_args args = { + .flags = (flags & ~CSIGNAL), + .exit_signal = (flags & CSIGNAL), + .tls = regs->u_regs[UREG_I3], + }; + +#ifdef CONFIG_COMPAT + if (test_thread_flag(TIF_32BIT)) { + args.pidfd = compat_ptr(regs->u_regs[UREG_I2]); + args.child_tid = compat_ptr(regs->u_regs[UREG_I4]); + args.parent_tid = compat_ptr(regs->u_regs[UREG_I2]); + } else +#endif + { + args.pidfd = (int __user *)regs->u_regs[UREG_I2]; + args.child_tid = (int __user *)regs->u_regs[UREG_I4]; + args.parent_tid = (int __user *)regs->u_regs[UREG_I2]; + } + + /* Did userspace give setup a separate stack for the child or are we + * reusing the parent's? + */ + if (regs->u_regs[UREG_I1]) + args.stack = regs->u_regs[UREG_I1]; + else + args.stack = regs->u_regs[UREG_FP]; + + ret = kernel_clone(&args); + + /* If we get an error and potentially restart the system + * call, we're screwed because copy_thread() clobbered + * the parent's %o1. So detect that case and restore it + * here. + */ + if ((unsigned long)ret >= -ERESTART_RESTARTBLOCK) + regs->u_regs[UREG_I1] = orig_i1; + + return ret; +} diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c index fdd819dfdacf..5a28c0e91bf1 100644 --- a/arch/sparc/kernel/process_32.c +++ b/arch/sparc/kernel/process_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* linux/arch/sparc/kernel/process.c * * Copyright (C) 1995, 2008 David S. Miller (davem@davemloft.net) @@ -7,12 +8,13 @@ /* * This file handles the architecture-dependent parts of process handling.. */ - -#include <stdarg.h> - +#include <linux/elfcore.h> #include <linux/errno.h> #include <linux/module.h> #include <linux/sched.h> +#include <linux/sched/debug.h> +#include <linux/sched/task.h> +#include <linux/sched/task_stack.h> #include <linux/kernel.h> #include <linux/mm.h> #include <linux/stddef.h> @@ -22,15 +24,13 @@ #include <linux/reboot.h> #include <linux/delay.h> #include <linux/pm.h> -#include <linux/init.h> #include <linux/slab.h> +#include <linux/cpu.h> #include <asm/auxio.h> #include <asm/oplib.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/page.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> #include <asm/delay.h> #include <asm/processor.h> #include <asm/psr.h> @@ -39,6 +39,8 @@ #include <asm/unistd.h> #include <asm/setup.h> +#include "kernel.h" + /* * Power management idle function * Set in pm platform drivers (apc.c and pmc.c) @@ -69,7 +71,6 @@ void arch_cpu_idle(void) { if (sparc_idle) (*sparc_idle)(); - local_irq_enable(); } /* XXX cli/sti -> local_irq_xxx here, check this works once SMP is fixed. */ @@ -103,8 +104,12 @@ void machine_restart(char * cmd) void machine_power_off(void) { if (auxio_power_register && - (strcmp(of_console_device->type, "serial") || scons_pwroff)) - *auxio_power_register |= AUXIO_POWER_OFF; + (!of_node_is_type(of_console_device, "serial") || scons_pwroff)) { + u8 power_register = sbus_readb(auxio_power_register); + power_register |= AUXIO_POWER_OFF; + sbus_writeb(power_register, auxio_power_register); + } + machine_halt(); } @@ -134,10 +139,10 @@ void show_regs(struct pt_regs *r) } /* - * The show_stack is an external API which we do not use ourselves. + * The show_stack() is external API which we do not use ourselves. * The oops is printed in die_if_kernel. */ -void show_stack(struct task_struct *tsk, unsigned long *_ksp) +void show_stack(struct task_struct *tsk, unsigned long *_ksp, const char *loglvl) { unsigned long pc, fp; unsigned long task_base; @@ -159,39 +164,31 @@ void show_stack(struct task_struct *tsk, unsigned long *_ksp) break; rw = (struct reg_window32 *) fp; pc = rw->ins[7]; - printk("[%08lx : ", pc); - printk("%pS ] ", (void *) pc); + printk("%s[%08lx : ", loglvl, pc); + printk("%s%pS ] ", loglvl, (void *) pc); fp = rw->ins[6]; } while (++count < 16); - printk("\n"); -} - -/* - * Note: sparc64 has a pretty intricated thread_saved_pc, check it out. - */ -unsigned long thread_saved_pc(struct task_struct *tsk) -{ - return task_thread_info(tsk)->kpc; + printk("%s\n", loglvl); } /* * Free current thread data structures etc.. */ -void exit_thread(void) +void exit_thread(struct task_struct *tsk) { #ifndef CONFIG_SMP - if(last_task_used_math == current) { + if (last_task_used_math == tsk) { #else - if (test_thread_flag(TIF_USEDFPU)) { + if (test_tsk_thread_flag(tsk, TIF_USEDFPU)) { #endif /* Keep process from leaving FPU in a bogon state. */ put_psr(get_psr() | PSR_EF); - fpsave(¤t->thread.float_regs[0], ¤t->thread.fsr, - ¤t->thread.fpqueue[0], ¤t->thread.fpqdepth); + fpsave(&tsk->thread.float_regs[0], &tsk->thread.fsr, + &tsk->thread.fpqueue[0], &tsk->thread.fpqdepth); #ifndef CONFIG_SMP last_task_used_math = NULL; #else - clear_thread_flag(TIF_USEDFPU); + clear_ti_thread_flag(task_thread_info(tsk), TIF_USEDFPU); #endif } } @@ -215,16 +212,6 @@ void flush_thread(void) clear_thread_flag(TIF_USEDFPU); #endif } - - /* This task is no longer a kernel thread. */ - if (current->thread.flags & SPARC_FLAG_KTHREAD) { - current->thread.flags &= ~SPARC_FLAG_KTHREAD; - - /* We must fixup kregs as well. */ - /* XXX This was not fixed for ti for a while, worked. Unused? */ - current->thread.kregs = (struct pt_regs *) - (task_stack_page(current) + (THREAD_SIZE - TRACEREG_SZ)); - } } static inline struct sparc_stackf __user * @@ -255,33 +242,6 @@ clone_stackframe(struct sparc_stackf __user *dst, return sp; } -asmlinkage int sparc_do_fork(unsigned long clone_flags, - unsigned long stack_start, - struct pt_regs *regs, - unsigned long stack_size) -{ - unsigned long parent_tid_ptr, child_tid_ptr; - unsigned long orig_i1 = regs->u_regs[UREG_I1]; - long ret; - - parent_tid_ptr = regs->u_regs[UREG_I2]; - child_tid_ptr = regs->u_regs[UREG_I4]; - - ret = do_fork(clone_flags, stack_start, stack_size, - (int __user *) parent_tid_ptr, - (int __user *) child_tid_ptr); - - /* If we get an error and potentially restart the system - * call, we're screwed because copy_thread() clobbered - * the parent's %o1. So detect that case and restore it - * here. - */ - if ((unsigned long)ret >= -ERESTART_RESTARTBLOCK) - regs->u_regs[UREG_I1] = orig_i1; - - return ret; -} - /* Copy a Sparc thread. The fork() return value conventions * under SunOS are nothing short of bletcherous: * Parent --> %o0 == childs pid, %o1 == 0 @@ -298,9 +258,11 @@ asmlinkage int sparc_do_fork(unsigned long clone_flags, extern void ret_from_fork(void); extern void ret_from_kernel_thread(void); -int copy_thread(unsigned long clone_flags, unsigned long sp, - unsigned long arg, struct task_struct *p) +int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) { + u64 clone_flags = args->flags; + unsigned long sp = args->stack; + unsigned long tls = args->tls; struct thread_info *ti = task_thread_info(p); struct pt_regs *childregs, *regs = current_pt_regs(); char *new_stack; @@ -326,24 +288,22 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, childregs = (struct pt_regs *) (new_stack + STACKFRAME_SZ); /* - * A new process must start with interrupts closed in 2.5, - * because this is how Mingo's scheduler works (see schedule_tail - * and finish_arch_switch). If we do not do it, a timer interrupt hits - * before we unlock, attempts to re-take the rq->lock, and then we die. - * Thus, kpsr|=PSR_PIL. + * A new process must start with interrupts disabled, see schedule_tail() + * and finish_task_switch(). (If we do not do it and if a timer interrupt + * hits before we unlock and attempts to take the rq->lock, we deadlock.) + * + * Thus, kpsr |= PSR_PIL. */ ti->ksp = (unsigned long) new_stack; p->thread.kregs = childregs; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(args->fn)) { extern int nwindows; unsigned long psr; memset(new_stack, 0, STACKFRAME_SZ + TRACEREG_SZ); - p->thread.flags |= SPARC_FLAG_KTHREAD; - p->thread.current_ds = KERNEL_DS; ti->kpc = (((unsigned long) ret_from_kernel_thread) - 0x8); - childregs->u_regs[UREG_G1] = sp; /* function */ - childregs->u_regs[UREG_G2] = arg; + childregs->u_regs[UREG_G1] = (unsigned long) args->fn; + childregs->u_regs[UREG_G2] = (unsigned long) args->fn_arg; psr = childregs->psr = get_psr(); ti->kpsr = psr | PSR_PIL; ti->kwim = 1 << (((psr & PSR_CWP) + 1) % nwindows); @@ -351,8 +311,6 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, } memcpy(new_stack, (char *)regs - STACKFRAME_SZ, STACKFRAME_SZ + TRACEREG_SZ); childregs->u_regs[UREG_FP] = sp; - p->thread.flags &= ~SPARC_FLAG_KTHREAD; - p->thread.current_ds = USER_DS; ti->kpc = (((unsigned long) ret_from_fork) - 0x8); ti->kpsr = current->thread.fork_kpsr | PSR_PIL; ti->kwim = current->thread.fork_kwim; @@ -401,61 +359,12 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, regs->u_regs[UREG_I1] = 0; if (clone_flags & CLONE_SETTLS) - childregs->u_regs[UREG_G7] = regs->u_regs[UREG_I3]; + childregs->u_regs[UREG_G7] = tls; return 0; } -/* - * fill in the fpu structure for a core dump. - */ -int dump_fpu (struct pt_regs * regs, elf_fpregset_t * fpregs) -{ - if (used_math()) { - memset(fpregs, 0, sizeof(*fpregs)); - fpregs->pr_q_entrysize = 8; - return 1; - } -#ifdef CONFIG_SMP - if (test_thread_flag(TIF_USEDFPU)) { - put_psr(get_psr() | PSR_EF); - fpsave(¤t->thread.float_regs[0], ¤t->thread.fsr, - ¤t->thread.fpqueue[0], ¤t->thread.fpqdepth); - if (regs != NULL) { - regs->psr &= ~(PSR_EF); - clear_thread_flag(TIF_USEDFPU); - } - } -#else - if (current == last_task_used_math) { - put_psr(get_psr() | PSR_EF); - fpsave(¤t->thread.float_regs[0], ¤t->thread.fsr, - ¤t->thread.fpqueue[0], ¤t->thread.fpqdepth); - if (regs != NULL) { - regs->psr &= ~(PSR_EF); - last_task_used_math = NULL; - } - } -#endif - memcpy(&fpregs->pr_fr.pr_regs[0], - ¤t->thread.float_regs[0], - (sizeof(unsigned long) * 32)); - fpregs->pr_fsr = current->thread.fsr; - fpregs->pr_qcnt = current->thread.fpqdepth; - fpregs->pr_q_entrysize = 8; - fpregs->pr_en = 1; - if(fpregs->pr_qcnt != 0) { - memcpy(&fpregs->pr_q[0], - ¤t->thread.fpqueue[0], - sizeof(struct fpq) * fpregs->pr_qcnt); - } - /* Zero out the rest. */ - memset(&fpregs->pr_q[fpregs->pr_qcnt], 0, - sizeof(struct fpq) * (32 - fpregs->pr_qcnt)); - return 1; -} - -unsigned long get_wchan(struct task_struct *task) +unsigned long __get_wchan(struct task_struct *task) { unsigned long pc, fp, bias = 0; unsigned long task_base = (unsigned long) task; @@ -463,10 +372,6 @@ unsigned long get_wchan(struct task_struct *task) struct reg_window32 *rw; int count = 0; - if (!task || task == current || - task->state == TASK_RUNNING) - goto out; - fp = task_thread_info(task)->ksp + bias; do { /* Bogus frame pointer? */ diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index baebab215492..25781923788a 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* arch/sparc64/kernel/process.c * * Copyright (C) 1995, 1996, 2008 David S. Miller (davem@davemloft.net) @@ -8,12 +9,12 @@ /* * This file handles the architecture-dependent parts of process handling.. */ - -#include <stdarg.h> - #include <linux/errno.h> #include <linux/export.h> #include <linux/sched.h> +#include <linux/sched/debug.h> +#include <linux/sched/task.h> +#include <linux/sched/task_stack.h> #include <linux/kernel.h> #include <linux/mm.h> #include <linux/fs.h> @@ -31,11 +32,12 @@ #include <linux/elfcore.h> #include <linux/sysrq.h> #include <linux/nmi.h> +#include <linux/context_tracking.h> +#include <linux/signal.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/page.h> #include <asm/pgalloc.h> -#include <asm/pgtable.h> #include <asm/processor.h> #include <asm/pstate.h> #include <asm/elf.h> @@ -60,6 +62,8 @@ void arch_cpu_idle(void) } else { unsigned long pstate; + raw_local_irq_enable(); + /* The sun4v sleeping code requires that we have PSTATE.IE cleared over * the cpu sleep hypervisor call. */ @@ -70,8 +74,13 @@ void arch_cpu_idle(void) : "=&r" (pstate) : "i" (PSTATE_IE)); - if (!need_resched() && !cpu_is_offline(smp_processor_id())) + if (!need_resched() && !cpu_is_offline(smp_processor_id())) { sun4v_cpu_yield(); + /* If resumed by cpu_poke then we need to explicitly + * call scheduler_ipi(). + */ + scheduler_poke(); + } /* Re-enable interrupts. */ __asm__ __volatile__( @@ -80,12 +89,13 @@ void arch_cpu_idle(void) "wrpr %0, %%g0, %%pstate" : "=&r" (pstate) : "i" (PSTATE_IE)); + + raw_local_irq_disable(); } - local_irq_enable(); } #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead() +void __noreturn arch_cpu_idle_dead(void) { sched_preempt_enable_no_resched(); cpu_play_dead(); @@ -97,18 +107,13 @@ static void show_regwindow32(struct pt_regs *regs) { struct reg_window32 __user *rw; struct reg_window32 r_w; - mm_segment_t old_fs; __asm__ __volatile__ ("flushw"); - rw = compat_ptr((unsigned)regs->u_regs[14]); - old_fs = get_fs(); - set_fs (USER_DS); + rw = compat_ptr((unsigned int)regs->u_regs[14]); if (copy_from_user (&r_w, rw, sizeof(r_w))) { - set_fs (old_fs); return; } - set_fs (old_fs); printk("l0: %08x l1: %08x l2: %08x l3: %08x " "l4: %08x l5: %08x l6: %08x l7: %08x\n", r_w.locals[0], r_w.locals[1], r_w.locals[2], r_w.locals[3], @@ -127,7 +132,6 @@ static void show_regwindow(struct pt_regs *regs) struct reg_window __user *rw; struct reg_window *rwk; struct reg_window r_w; - mm_segment_t old_fs; if ((regs->tstate & TSTATE_PRIV) || !(test_thread_flag(TIF_32BIT))) { __asm__ __volatile__ ("flushw"); @@ -136,14 +140,10 @@ static void show_regwindow(struct pt_regs *regs) rwk = (struct reg_window *) (regs->u_regs[14] + STACK_BIAS); if (!(regs->tstate & TSTATE_PRIV)) { - old_fs = get_fs(); - set_fs (USER_DS); if (copy_from_user (&r_w, rw, sizeof(r_w))) { - set_fs (old_fs); return; } rwk = &r_w; - set_fs (old_fs); } } else { show_regwindow32(regs); @@ -182,7 +182,7 @@ void show_regs(struct pt_regs *regs) regs->u_regs[15]); printk("RPC: <%pS>\n", (void *) regs->u_regs[15]); show_regwindow(regs); - show_stack(current, (unsigned long *) regs->u_regs[UREG_FP]); + show_stack(current, (unsigned long *)regs->u_regs[UREG_FP], KERN_DEFAULT); } union global_cpu_snapshot global_cpu_snapshot[NR_CPUS]; @@ -236,7 +236,7 @@ static void __global_reg_poll(struct global_reg_snapshot *gp) } } -void arch_trigger_all_cpu_backtrace(void) +void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) { struct thread_info *tp = current_thread_info(); struct pt_regs *regs = get_irq_regs(); @@ -248,16 +248,22 @@ void arch_trigger_all_cpu_backtrace(void) spin_lock_irqsave(&global_cpu_snapshot_lock, flags); - memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); - this_cpu = raw_smp_processor_id(); - __global_reg_self(tp, regs, this_cpu); + memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); + + if (cpumask_test_cpu(this_cpu, mask) && this_cpu != exclude_cpu) + __global_reg_self(tp, regs, this_cpu); smp_fetch_global_regs(); - for_each_online_cpu(cpu) { - struct global_reg_snapshot *gp = &global_cpu_snapshot[cpu].reg; + for_each_cpu(cpu, mask) { + struct global_reg_snapshot *gp; + + if (cpu == exclude_cpu) + continue; + + gp = &global_cpu_snapshot[cpu].reg; __global_reg_poll(gp); @@ -278,6 +284,8 @@ void arch_trigger_all_cpu_backtrace(void) printk(" TPC[%lx] O7[%lx] I7[%lx] RPC[%lx]\n", gp->tpc, gp->o7, gp->i7, gp->rpc); } + + touch_nmi_watchdog(); } memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); @@ -287,12 +295,12 @@ void arch_trigger_all_cpu_backtrace(void) #ifdef CONFIG_MAGIC_SYSRQ -static void sysrq_handle_globreg(int key) +static void sysrq_handle_globreg(u8 key) { - arch_trigger_all_cpu_backtrace(); + trigger_all_cpu_backtrace(); } -static struct sysrq_key_op sparc_globalreg_op = { +static const struct sysrq_key_op sparc_globalreg_op = { .handler = sysrq_handle_globreg, .help_msg = "global-regs(y)", .action_msg = "Show Global CPU Regs", @@ -303,6 +311,9 @@ static void __global_pmu_self(int this_cpu) struct global_pmu_snapshot *pp; int i, num; + if (!pcr_ops) + return; + pp = &global_cpu_snapshot[this_cpu].pmu; num = 1; @@ -350,6 +361,8 @@ static void pmu_snapshot_all_cpus(void) (cpu == this_cpu ? '*' : ' '), cpu, pp->pcr[0], pp->pcr[1], pp->pcr[2], pp->pcr[3], pp->pic[0], pp->pic[1], pp->pic[2], pp->pic[3]); + + touch_nmi_watchdog(); } memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); @@ -357,12 +370,12 @@ static void pmu_snapshot_all_cpus(void) spin_unlock_irqrestore(&global_cpu_snapshot_lock, flags); } -static void sysrq_handle_globpmu(int key) +static void sysrq_handle_globpmu(u8 key) { pmu_snapshot_all_cpus(); } -static struct sysrq_key_op sparc_globalpmu_op = { +static const struct sysrq_key_op sparc_globalpmu_op = { .handler = sysrq_handle_globpmu, .help_msg = "global-pmu(x)", .action_msg = "Show Global PMU Regs", @@ -381,29 +394,10 @@ core_initcall(sparc_sysrq_init); #endif -unsigned long thread_saved_pc(struct task_struct *tsk) -{ - struct thread_info *ti = task_thread_info(tsk); - unsigned long ret = 0xdeadbeefUL; - - if (ti && ti->ksp) { - unsigned long *sp; - sp = (unsigned long *)(ti->ksp + STACK_BIAS); - if (((unsigned long)sp & (sizeof(long) - 1)) == 0UL && - sp[14]) { - unsigned long *fp; - fp = (unsigned long *)(sp[14] + STACK_BIAS); - if (((unsigned long)fp & (sizeof(long) - 1)) == 0UL) - ret = fp[15]; - } - } - return ret; -} - /* Free current thread data structures etc.. */ -void exit_thread(void) +void exit_thread(struct task_struct *tsk) { - struct thread_info *t = current_thread_info(); + struct thread_info *t = task_thread_info(tsk); if (t->utraps) { if (t->utraps[0] < 2) @@ -452,7 +446,7 @@ static unsigned long clone_stackframe(unsigned long csp, unsigned long psp) distance = fp - psp; rval = (csp - distance); - if (copy_in_user((void __user *) rval, (void __user *) psp, distance)) + if (raw_copy_in_user((void __user *)rval, (void __user *)psp, distance)) rval = 0; else if (!stack_64bit) { if (put_user(((u32)csp), @@ -512,17 +506,15 @@ void synchronize_user_stack(void) static void stack_unaligned(unsigned long sp) { - siginfo_t info; - - info.si_signo = SIGBUS; - info.si_errno = 0; - info.si_code = BUS_ADRALN; - info.si_addr = (void __user *) sp; - info.si_trapno = 0; - force_sig_info(SIGBUS, &info, current); + force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) sp); } -void fault_in_user_windows(void) +static const char uwfault32[] = KERN_INFO \ + "%s[%d]: bad register window fault: SP %08lx (orig_sp %08lx) TPC %08lx O7 %08lx\n"; +static const char uwfault64[] = KERN_INFO \ + "%s[%d]: bad register window fault: SP %016lx (orig_sp %016lx) TPC %08lx O7 %016lx\n"; + +void fault_in_user_windows(struct pt_regs *regs) { struct thread_info *t = current_thread_info(); unsigned long window; @@ -535,9 +527,9 @@ void fault_in_user_windows(void) do { struct reg_window *rwin = &t->reg_window[window]; int winsize = sizeof(struct reg_window); - unsigned long sp; + unsigned long sp, orig_sp; - sp = t->rwbuf_stkptrs[window]; + orig_sp = sp = t->rwbuf_stkptrs[window]; if (test_thread_64bit_stack(sp)) sp += STACK_BIAS; @@ -548,8 +540,16 @@ void fault_in_user_windows(void) stack_unaligned(sp); if (unlikely(copy_to_user((char __user *)sp, - rwin, winsize))) + rwin, winsize))) { + if (show_unhandled_signals) + printk_ratelimited(is_compat_task() ? + uwfault32 : uwfault64, + current->comm, current->pid, + sp, orig_sp, + regs->tpc, + regs->u_regs[UREG_I7]); goto barf; + } } while (window--); } set_thread_wsaved(0); @@ -557,41 +557,7 @@ void fault_in_user_windows(void) barf: set_thread_wsaved(window + 1); - do_exit(SIGILL); -} - -asmlinkage long sparc_do_fork(unsigned long clone_flags, - unsigned long stack_start, - struct pt_regs *regs, - unsigned long stack_size) -{ - int __user *parent_tid_ptr, *child_tid_ptr; - unsigned long orig_i1 = regs->u_regs[UREG_I1]; - long ret; - -#ifdef CONFIG_COMPAT - if (test_thread_flag(TIF_32BIT)) { - parent_tid_ptr = compat_ptr(regs->u_regs[UREG_I2]); - child_tid_ptr = compat_ptr(regs->u_regs[UREG_I4]); - } else -#endif - { - parent_tid_ptr = (int __user *) regs->u_regs[UREG_I2]; - child_tid_ptr = (int __user *) regs->u_regs[UREG_I4]; - } - - ret = do_fork(clone_flags, stack_start, stack_size, - parent_tid_ptr, child_tid_ptr); - - /* If we get an error and potentially restart the system - * call, we're screwed because copy_thread() clobbered - * the parent's %o1. So detect that case and restore it - * here. - */ - if ((unsigned long)ret >= -ERESTART_RESTARTBLOCK) - regs->u_regs[UREG_I1] = orig_i1; - - return ret; + force_sig(SIGSEGV); } /* Copy a Sparc thread. The fork() return value conventions @@ -599,9 +565,11 @@ asmlinkage long sparc_do_fork(unsigned long clone_flags, * Parent --> %o0 == childs pid, %o1 == 0 * Child --> %o0 == parents pid, %o1 == 1 */ -int copy_thread(unsigned long clone_flags, unsigned long sp, - unsigned long arg, struct task_struct *p) +int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) { + u64 clone_flags = args->flags; + unsigned long sp = args->stack; + unsigned long tls = args->tls; struct thread_info *t = task_thread_info(p); struct pt_regs *regs = current_pt_regs(); struct sparc_stackf *parent_sf; @@ -619,13 +587,12 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, sizeof(struct sparc_stackf)); t->fpsaved[0] = 0; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(args->fn)) { memset(child_trap_frame, 0, child_stack_sz); __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] = (current_pt_regs()->tstate + 1) & TSTATE_CWP; - t->current_ds = ASI_P; - t->kregs->u_regs[UREG_G1] = sp; /* function */ - t->kregs->u_regs[UREG_G2] = arg; + t->kregs->u_regs[UREG_G1] = (unsigned long) args->fn; + t->kregs->u_regs[UREG_G2] = (unsigned long) args->fn_arg; return 0; } @@ -638,7 +605,6 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, t->kregs->u_regs[UREG_FP] = sp; __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] = (regs->tstate + 1) & TSTATE_CWP; - t->current_ds = ASI_AIUS; if (sp != regs->u_regs[UREG_FP]) { unsigned long csp; @@ -658,78 +624,37 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, regs->u_regs[UREG_I1] = 0; if (clone_flags & CLONE_SETTLS) - t->kregs->u_regs[UREG_G7] = regs->u_regs[UREG_I3]; + t->kregs->u_regs[UREG_G7] = tls; return 0; } -typedef struct { - union { - unsigned int pr_regs[32]; - unsigned long pr_dregs[16]; - } pr_fr; - unsigned int __unused; - unsigned int pr_fsr; - unsigned char pr_qcnt; - unsigned char pr_q_entrysize; - unsigned char pr_en; - unsigned int pr_q[64]; -} elf_fpregset_t32; - -/* - * fill in the fpu structure for a core dump. +/* TIF_MCDPER in thread info flags for current task is updated lazily upon + * a context switch. Update this flag in current task's thread flags + * before dup so the dup'd task will inherit the current TIF_MCDPER flag. */ -int dump_fpu (struct pt_regs * regs, elf_fpregset_t * fpregs) +int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { - unsigned long *kfpregs = current_thread_info()->fpregs; - unsigned long fprs = current_thread_info()->fpsaved[0]; + if (adi_capable()) { + register unsigned long tmp_mcdper; - if (test_thread_flag(TIF_32BIT)) { - elf_fpregset_t32 *fpregs32 = (elf_fpregset_t32 *)fpregs; - - if (fprs & FPRS_DL) - memcpy(&fpregs32->pr_fr.pr_regs[0], kfpregs, - sizeof(unsigned int) * 32); - else - memset(&fpregs32->pr_fr.pr_regs[0], 0, - sizeof(unsigned int) * 32); - fpregs32->pr_qcnt = 0; - fpregs32->pr_q_entrysize = 8; - memset(&fpregs32->pr_q[0], 0, - (sizeof(unsigned int) * 64)); - if (fprs & FPRS_FEF) { - fpregs32->pr_fsr = (unsigned int) current_thread_info()->xfsr[0]; - fpregs32->pr_en = 1; - } else { - fpregs32->pr_fsr = 0; - fpregs32->pr_en = 0; - } - } else { - if(fprs & FPRS_DL) - memcpy(&fpregs->pr_regs[0], kfpregs, - sizeof(unsigned int) * 32); - else - memset(&fpregs->pr_regs[0], 0, - sizeof(unsigned int) * 32); - if(fprs & FPRS_DU) - memcpy(&fpregs->pr_regs[16], kfpregs+16, - sizeof(unsigned int) * 32); + __asm__ __volatile__( + ".word 0x83438000\n\t" /* rd %mcdper, %g1 */ + "mov %%g1, %0\n\t" + : "=r" (tmp_mcdper) + : + : "g1"); + if (tmp_mcdper) + set_thread_flag(TIF_MCDPER); else - memset(&fpregs->pr_regs[16], 0, - sizeof(unsigned int) * 32); - if(fprs & FPRS_FEF) { - fpregs->pr_fsr = current_thread_info()->xfsr[0]; - fpregs->pr_gsr = current_thread_info()->gsr[0]; - } else { - fpregs->pr_fsr = fpregs->pr_gsr = 0; - } - fpregs->pr_fprs = fprs; + clear_thread_flag(TIF_MCDPER); } - return 1; + + *dst = *src; + return 0; } -EXPORT_SYMBOL(dump_fpu); -unsigned long get_wchan(struct task_struct *task) +unsigned long __get_wchan(struct task_struct *task) { unsigned long pc, fp, bias = 0; struct thread_info *tp; @@ -737,10 +662,6 @@ unsigned long get_wchan(struct task_struct *task) unsigned long ret = 0; int count = 0; - if (!task || task == current || - task->state == TASK_RUNNING) - goto out; - tp = task_thread_info(task); bias = STACK_BIAS; fp = task_thread_info(task)->ksp + bias; diff --git a/arch/sparc/kernel/prom.h b/arch/sparc/kernel/prom.h index cf5fe1c0b024..26a1cca7c761 100644 --- a/arch/sparc/kernel/prom.h +++ b/arch/sparc/kernel/prom.h @@ -1,10 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __PROM_H #define __PROM_H #include <linux/spinlock.h> #include <asm/prom.h> -extern void of_console_init(void); +void of_console_init(void); extern unsigned int prom_early_allocated; diff --git a/arch/sparc/kernel/prom_32.c b/arch/sparc/kernel/prom_32.c index b51cbb9e87dc..cd94f1e8d644 100644 --- a/arch/sparc/kernel/prom_32.c +++ b/arch/sparc/kernel/prom_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Procedures for creating, accessing and interpreting the device tree. * @@ -8,18 +9,13 @@ * {engebret|bergner}@us.ibm.com * * Adapted for sparc32 by David S. Miller davem@davemloft.net - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #include <linux/kernel.h> #include <linux/types.h> #include <linux/string.h> #include <linux/mm.h> -#include <linux/bootmem.h> +#include <linux/memblock.h> #include <asm/prom.h> #include <asm/oplib.h> @@ -32,9 +28,7 @@ void * __init prom_early_alloc(unsigned long size) { void *ret; - ret = __alloc_bootmem(size, SMP_CACHE_BYTES, 0UL); - if (ret != NULL) - memset(ret, 0, size); + ret = memblock_alloc_or_panic(size, SMP_CACHE_BYTES); prom_early_allocated += size; @@ -60,6 +54,7 @@ void * __init prom_early_alloc(unsigned long size) */ static void __init sparc32_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct linux_prom_registers *regs; struct property *rprop; @@ -69,13 +64,14 @@ static void __init sparc32_path_component(struct device_node *dp, char *tmp_buf) regs = rprop->value; sprintf(tmp_buf, "%s@%x,%x", - dp->name, + name, regs->which_io, regs->phys_addr); } /* "name@slot,offset" */ static void __init sbus_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct linux_prom_registers *regs; struct property *prop; @@ -85,7 +81,7 @@ static void __init sbus_path_component(struct device_node *dp, char *tmp_buf) regs = prop->value; sprintf(tmp_buf, "%s@%x,%x", - dp->name, + name, regs->which_io, regs->phys_addr); } @@ -93,6 +89,7 @@ static void __init sbus_path_component(struct device_node *dp, char *tmp_buf) /* "name@devnum[,func]" */ static void __init pci_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct linux_prom_pci_registers *regs; struct property *prop; unsigned int devfn; @@ -105,12 +102,12 @@ static void __init pci_path_component(struct device_node *dp, char *tmp_buf) devfn = (regs->phys_hi >> 8) & 0xff; if (devfn & 0x07) { sprintf(tmp_buf, "%s@%x,%x", - dp->name, + name, devfn >> 3, devfn & 0x07); } else { sprintf(tmp_buf, "%s@%x", - dp->name, + name, devfn >> 3); } } @@ -118,6 +115,7 @@ static void __init pci_path_component(struct device_node *dp, char *tmp_buf) /* "name@addrhi,addrlo" */ static void __init ebus_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct linux_prom_registers *regs; struct property *prop; @@ -128,15 +126,17 @@ static void __init ebus_path_component(struct device_node *dp, char *tmp_buf) regs = prop->value; sprintf(tmp_buf, "%s@%x,%x", - dp->name, + name, regs->which_io, regs->phys_addr); } -/* "name:vendor:device@irq,addrlo" */ +/* "name@irq,addrlo" */ static void __init ambapp_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct amba_prom_registers *regs; - unsigned int *intr, *device, *vendor, reg0; + unsigned int *intr; + unsigned int reg0; struct property *prop; int interrupt = 0; @@ -158,18 +158,7 @@ static void __init ambapp_path_component(struct device_node *dp, char *tmp_buf) else intr = prop->value; - prop = of_find_property(dp, "vendor", NULL); - if (!prop) - return; - vendor = prop->value; - prop = of_find_property(dp, "device", NULL); - if (!prop) - return; - device = prop->value; - - sprintf(tmp_buf, "%s:%d:%d@%x,%x", - dp->name, *vendor, *device, - *intr, reg0); + sprintf(tmp_buf, "%s@%x,%x", name, *intr, reg0); } static void __init __build_path_component(struct device_node *dp, char *tmp_buf) @@ -177,14 +166,14 @@ static void __init __build_path_component(struct device_node *dp, char *tmp_buf) struct device_node *parent = dp->parent; if (parent != NULL) { - if (!strcmp(parent->type, "pci") || - !strcmp(parent->type, "pciex")) + if (of_node_is_type(parent, "pci") || + of_node_is_type(parent, "pciex")) return pci_path_component(dp, tmp_buf); - if (!strcmp(parent->type, "sbus")) + if (of_node_is_type(parent, "sbus")) return sbus_path_component(dp, tmp_buf); - if (!strcmp(parent->type, "ebus")) + if (of_node_is_type(parent, "ebus")) return ebus_path_component(dp, tmp_buf); - if (!strcmp(parent->type, "ambapp")) + if (of_node_is_type(parent, "ambapp")) return ambapp_path_component(dp, tmp_buf); /* "isa" is handled with platform naming */ @@ -196,15 +185,18 @@ static void __init __build_path_component(struct device_node *dp, char *tmp_buf) char * __init build_path_component(struct device_node *dp) { + const char *name = of_get_property(dp, "name", NULL); char tmp_buf[64], *n; + size_t n_sz; tmp_buf[0] = '\0'; __build_path_component(dp, tmp_buf); if (tmp_buf[0] == '\0') - strcpy(tmp_buf, dp->name); + strscpy(tmp_buf, name); - n = prom_early_alloc(strlen(tmp_buf) + 1); - strcpy(n, tmp_buf); + n_sz = strlen(tmp_buf) + 1; + n = prom_early_alloc(n_sz); + strscpy(n, tmp_buf, n_sz); return n; } @@ -214,13 +206,14 @@ extern void restore_current(void); void __init of_console_init(void) { char *msg = "OF stdout device is: %s\n"; + const size_t of_console_path_sz = 256; struct device_node *dp; unsigned long flags; const char *type; phandle node; int skip, tmp, fd; - of_console_path = prom_early_alloc(256); + of_console_path = prom_early_alloc(of_console_path_sz); switch (prom_vers) { case PROM_V0: @@ -232,7 +225,7 @@ void __init of_console_init(void) case PROMDEV_TTYB: skip = 1; - /* FALLTHRU */ + fallthrough; case PROMDEV_TTYA: type = "serial"; @@ -255,7 +248,7 @@ void __init of_console_init(void) } of_console_device = dp; - strcpy(of_console_path, dp->full_name); + sprintf(of_console_path, "%pOF", dp); if (!strcmp(type, "serial")) { strcat(of_console_path, (skip ? ":b" : ":a")); @@ -278,15 +271,9 @@ void __init of_console_init(void) prom_halt(); } dp = of_find_node_by_phandle(node); - type = of_get_property(dp, "device_type", NULL); - - if (!type) { - prom_printf("Console stdout lacks " - "device_type property.\n"); - prom_halt(); - } - if (strcmp(type, "display") && strcmp(type, "serial")) { + if (!of_node_is_type(dp, "display") && + !of_node_is_type(dp, "serial")) { prom_printf("Console device_type is neither display " "nor serial.\n"); prom_halt(); @@ -295,7 +282,7 @@ void __init of_console_init(void) of_console_device = dp; if (prom_vers == PROM_V2) { - strcpy(of_console_path, dp->full_name); + sprintf(of_console_path, "%pOF", dp); switch (*romvec->pv_stdout) { case PROMDEV_TTYA: strcat(of_console_path, ":a"); @@ -313,7 +300,7 @@ void __init of_console_init(void) prom_printf("No stdout-path in root node.\n"); prom_halt(); } - strcpy(of_console_path, path); + strscpy(of_console_path, path, of_console_path_sz); } break; } diff --git a/arch/sparc/kernel/prom_64.c b/arch/sparc/kernel/prom_64.c index d397d7fc5c28..aa4799cbb9c1 100644 --- a/arch/sparc/kernel/prom_64.c +++ b/arch/sparc/kernel/prom_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Procedures for creating, accessing and interpreting the device tree. * @@ -8,18 +9,14 @@ * {engebret|bergner}@us.ibm.com * * Adapted for sparc64 by David S. Miller davem@davemloft.net - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ +#include <linux/memblock.h> #include <linux/kernel.h> -#include <linux/types.h> #include <linux/string.h> +#include <linux/types.h> +#include <linux/cpu.h> #include <linux/mm.h> -#include <linux/memblock.h> #include <linux/of.h> #include <asm/prom.h> @@ -33,16 +30,13 @@ void * __init prom_early_alloc(unsigned long size) { - unsigned long paddr = memblock_alloc(size, SMP_CACHE_BYTES); - void *ret; + void *ret = memblock_alloc(size, SMP_CACHE_BYTES); - if (!paddr) { + if (!ret) { prom_printf("prom_early_alloc(%lu) failed\n", size); prom_halt(); } - ret = __va(paddr); - memset(ret, 0, size); prom_early_allocated += size; return ret; @@ -71,6 +65,7 @@ void * __init prom_early_alloc(unsigned long size) */ static void __init sun4v_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct linux_prom64_registers *regs; struct property *rprop; u32 high_bits, low_bits, type; @@ -82,7 +77,7 @@ static void __init sun4v_path_component(struct device_node *dp, char *tmp_buf) regs = rprop->value; if (!of_node_is_root(dp->parent)) { sprintf(tmp_buf, "%s@%x,%x", - dp->name, + name, (unsigned int) (regs->phys_addr >> 32UL), (unsigned int) (regs->phys_addr & 0xffffffffUL)); return; @@ -97,21 +92,22 @@ static void __init sun4v_path_component(struct device_node *dp, char *tmp_buf) if (low_bits) sprintf(tmp_buf, "%s@%s%x,%x", - dp->name, prefix, + name, prefix, high_bits, low_bits); else sprintf(tmp_buf, "%s@%s%x", - dp->name, + name, prefix, high_bits); } else if (type == 12) { sprintf(tmp_buf, "%s@%x", - dp->name, high_bits); + name, high_bits); } } static void __init sun4u_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct linux_prom64_registers *regs; struct property *prop; @@ -122,7 +118,7 @@ static void __init sun4u_path_component(struct device_node *dp, char *tmp_buf) regs = prop->value; if (!of_node_is_root(dp->parent)) { sprintf(tmp_buf, "%s@%x,%x", - dp->name, + name, (unsigned int) (regs->phys_addr >> 32UL), (unsigned int) (regs->phys_addr & 0xffffffffUL)); return; @@ -138,7 +134,7 @@ static void __init sun4u_path_component(struct device_node *dp, char *tmp_buf) mask = 0x7fffff; sprintf(tmp_buf, "%s@%x,%x", - dp->name, + name, *(u32 *)prop->value, (unsigned int) (regs->phys_addr & mask)); } @@ -147,6 +143,7 @@ static void __init sun4u_path_component(struct device_node *dp, char *tmp_buf) /* "name@slot,offset" */ static void __init sbus_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct linux_prom_registers *regs; struct property *prop; @@ -156,7 +153,7 @@ static void __init sbus_path_component(struct device_node *dp, char *tmp_buf) regs = prop->value; sprintf(tmp_buf, "%s@%x,%x", - dp->name, + name, regs->which_io, regs->phys_addr); } @@ -164,6 +161,7 @@ static void __init sbus_path_component(struct device_node *dp, char *tmp_buf) /* "name@devnum[,func]" */ static void __init pci_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct linux_prom_pci_registers *regs; struct property *prop; unsigned int devfn; @@ -176,12 +174,12 @@ static void __init pci_path_component(struct device_node *dp, char *tmp_buf) devfn = (regs->phys_hi >> 8) & 0xff; if (devfn & 0x07) { sprintf(tmp_buf, "%s@%x,%x", - dp->name, + name, devfn >> 3, devfn & 0x07); } else { sprintf(tmp_buf, "%s@%x", - dp->name, + name, devfn >> 3); } } @@ -189,6 +187,7 @@ static void __init pci_path_component(struct device_node *dp, char *tmp_buf) /* "name@UPA_PORTID,offset" */ static void __init upa_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct linux_prom64_registers *regs; struct property *prop; @@ -203,7 +202,7 @@ static void __init upa_path_component(struct device_node *dp, char *tmp_buf) return; sprintf(tmp_buf, "%s@%x,%x", - dp->name, + name, *(u32 *) prop->value, (unsigned int) (regs->phys_addr & 0xffffffffUL)); } @@ -211,6 +210,7 @@ static void __init upa_path_component(struct device_node *dp, char *tmp_buf) /* "name@reg" */ static void __init vdev_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct property *prop; u32 *regs; @@ -220,12 +220,13 @@ static void __init vdev_path_component(struct device_node *dp, char *tmp_buf) regs = prop->value; - sprintf(tmp_buf, "%s@%x", dp->name, *regs); + sprintf(tmp_buf, "%s@%x", name, *regs); } /* "name@addrhi,addrlo" */ static void __init ebus_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct linux_prom64_registers *regs; struct property *prop; @@ -236,7 +237,7 @@ static void __init ebus_path_component(struct device_node *dp, char *tmp_buf) regs = prop->value; sprintf(tmp_buf, "%s@%x,%x", - dp->name, + name, (unsigned int) (regs->phys_addr >> 32UL), (unsigned int) (regs->phys_addr & 0xffffffffUL)); } @@ -244,6 +245,7 @@ static void __init ebus_path_component(struct device_node *dp, char *tmp_buf) /* "name@bus,addr" */ static void __init i2c_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct property *prop; u32 *regs; @@ -257,12 +259,13 @@ static void __init i2c_path_component(struct device_node *dp, char *tmp_buf) * property of the i2c bus node etc. etc. */ sprintf(tmp_buf, "%s@%x,%x", - dp->name, regs[0], regs[1]); + name, regs[0], regs[1]); } /* "name@reg0[,reg1]" */ static void __init usb_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct property *prop; u32 *regs; @@ -274,16 +277,17 @@ static void __init usb_path_component(struct device_node *dp, char *tmp_buf) if (prop->length == sizeof(u32) || regs[1] == 1) { sprintf(tmp_buf, "%s@%x", - dp->name, regs[0]); + name, regs[0]); } else { sprintf(tmp_buf, "%s@%x,%x", - dp->name, regs[0], regs[1]); + name, regs[0], regs[1]); } } /* "name@reg0reg1[,reg2reg3]" */ static void __init ieee1394_path_component(struct device_node *dp, char *tmp_buf) { + const char *name = of_get_property(dp, "name", NULL); struct property *prop; u32 *regs; @@ -295,10 +299,10 @@ static void __init ieee1394_path_component(struct device_node *dp, char *tmp_buf if (regs[2] || regs[3]) { sprintf(tmp_buf, "%s@%08x%08x,%04x%08x", - dp->name, regs[0], regs[1], regs[2], regs[3]); + name, regs[0], regs[1], regs[2], regs[3]); } else { sprintf(tmp_buf, "%s@%08x%08x", - dp->name, regs[0], regs[1]); + name, regs[0], regs[1]); } } @@ -307,37 +311,37 @@ static void __init __build_path_component(struct device_node *dp, char *tmp_buf) struct device_node *parent = dp->parent; if (parent != NULL) { - if (!strcmp(parent->type, "pci") || - !strcmp(parent->type, "pciex")) { + if (of_node_is_type(parent, "pci") || + of_node_is_type(parent, "pciex")) { pci_path_component(dp, tmp_buf); return; } - if (!strcmp(parent->type, "sbus")) { + if (of_node_is_type(parent, "sbus")) { sbus_path_component(dp, tmp_buf); return; } - if (!strcmp(parent->type, "upa")) { + if (of_node_is_type(parent, "upa")) { upa_path_component(dp, tmp_buf); return; } - if (!strcmp(parent->type, "ebus")) { + if (of_node_is_type(parent, "ebus")) { ebus_path_component(dp, tmp_buf); return; } - if (!strcmp(parent->name, "usb") || - !strcmp(parent->name, "hub")) { + if (of_node_name_eq(parent, "usb") || + of_node_name_eq(parent, "hub")) { usb_path_component(dp, tmp_buf); return; } - if (!strcmp(parent->type, "i2c")) { + if (of_node_is_type(parent, "i2c")) { i2c_path_component(dp, tmp_buf); return; } - if (!strcmp(parent->type, "firewire")) { + if (of_node_is_type(parent, "firewire")) { ieee1394_path_component(dp, tmp_buf); return; } - if (!strcmp(parent->type, "virtual-devices")) { + if (of_node_is_type(parent, "virtual-devices")) { vdev_path_component(dp, tmp_buf); return; } @@ -355,15 +359,18 @@ static void __init __build_path_component(struct device_node *dp, char *tmp_buf) char * __init build_path_component(struct device_node *dp) { + const char *name = of_get_property(dp, "name", NULL); char tmp_buf[64], *n; + size_t n_sz; tmp_buf[0] = '\0'; __build_path_component(dp, tmp_buf); if (tmp_buf[0] == '\0') - strcpy(tmp_buf, dp->name); + strscpy(tmp_buf, name); - n = prom_early_alloc(strlen(tmp_buf) + 1); - strcpy(n, tmp_buf); + n_sz = strlen(tmp_buf) + 1; + n = prom_early_alloc(n_sz); + strscpy(n, tmp_buf, n_sz); return n; } @@ -373,6 +380,59 @@ static const char *get_mid_prop(void) return (tlb_type == spitfire ? "upa-portid" : "portid"); } +bool arch_find_n_match_cpu_physical_id(struct device_node *cpun, + int cpu, unsigned int *thread) +{ + const char *mid_prop = get_mid_prop(); + int this_cpu_id; + + /* On hypervisor based platforms we interrogate the 'reg' + * property. On everything else we look for a 'upa-portid', + * 'portid', or 'cpuid' property. + */ + + if (tlb_type == hypervisor) { + struct property *prop = of_find_property(cpun, "reg", NULL); + u32 *regs; + + if (!prop) { + pr_warn("CPU node missing reg property\n"); + return false; + } + regs = prop->value; + this_cpu_id = regs[0] & 0x0fffffff; + } else { + this_cpu_id = of_getintprop_default(cpun, mid_prop, -1); + + if (this_cpu_id < 0) { + mid_prop = "cpuid"; + this_cpu_id = of_getintprop_default(cpun, mid_prop, -1); + } + if (this_cpu_id < 0) { + pr_warn("CPU node missing cpu ID property\n"); + return false; + } + } + if (this_cpu_id == cpu) { + if (thread) { + int proc_id = cpu_data(cpu).proc_id; + + /* On sparc64, the cpu thread information is obtained + * either from OBP or the machine description. We've + * actually probed this information already long before + * this interface gets called so instead of interrogating + * both the OF node and the MDESC again, just use what + * we discovered already. + */ + if (proc_id < 0) + proc_id = 0; + *thread = proc_id; + } + return true; + } + return false; +} + static void *of_iterate_over_cpus(void *(*func)(struct device_node *, int, int), int arg) { struct device_node *dp; @@ -425,7 +485,9 @@ static void *record_one_cpu(struct device_node *dp, int cpuid, int arg) ncpus_probed++; #ifdef CONFIG_SMP set_cpu_present(cpuid, true); - set_cpu_possible(cpuid, true); + + if (num_possible_cpus() < nr_cpu_ids) + set_cpu_possible(cpuid, true); #endif return NULL; } @@ -444,7 +506,7 @@ static void *fill_in_one_cpu(struct device_node *dp, int cpuid, int arg) struct device_node *portid_parent = NULL; int portid = -1; - if (of_find_property(dp, "cpuid", NULL)) { + if (of_property_present(dp, "cpuid")) { int limit = 2; portid_parent = dp; @@ -502,9 +564,6 @@ static void *fill_in_one_cpu(struct device_node *dp, int cpuid, int arg) cpu_data(cpuid).core_id = portid + 1; cpu_data(cpuid).proc_id = portid; -#ifdef CONFIG_SMP - sparc64_multi_core = 1; -#endif } else { cpu_data(cpuid).dcache_size = of_getintprop_default(dp, "dcache-size", 16 * 1024); @@ -543,7 +602,6 @@ void __init of_console_init(void) { char *msg = "OF stdout device is: %s\n"; struct device_node *dp; - const char *type; phandle node; of_console_path = prom_early_alloc(256); @@ -566,13 +624,8 @@ void __init of_console_init(void) } dp = of_find_node_by_phandle(node); - type = of_get_property(dp, "device_type", NULL); - if (!type) { - prom_printf("Console stdout lacks device_type property.\n"); - prom_halt(); - } - if (strcmp(type, "display") && strcmp(type, "serial")) { + if (!of_node_is_type(dp, "display") && !of_node_is_type(dp, "serial")) { prom_printf("Console device_type is neither display " "nor serial.\n"); prom_halt(); diff --git a/arch/sparc/kernel/prom_common.c b/arch/sparc/kernel/prom_common.c index 79cc0d1a477d..d258fd10db01 100644 --- a/arch/sparc/kernel/prom_common.c +++ b/arch/sparc/kernel/prom_common.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* prom_common.c: OF device tree support common code. * * Paul Mackerras August 1996. @@ -7,11 +8,6 @@ * {engebret|bergner}@us.ibm.com * * Adapted for sparc by David S. Miller davem@davemloft.net - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #include <linux/kernel.h> @@ -124,11 +120,14 @@ EXPORT_SYMBOL(of_find_in_proplist); */ static int __init handle_nextprop_quirks(char *buf, const char *name) { - if (!name || strlen(name) == 0) + size_t name_len; + + name_len = name ? strlen(name) : 0; + if (name_len == 0) return -1; #ifdef CONFIG_SPARC32 - strcpy(buf, name); + strscpy(buf, name, name_len + 1); #endif return 0; } diff --git a/arch/sparc/kernel/prom_irqtrans.c b/arch/sparc/kernel/prom_irqtrans.c index 40e4936bd479..5752bfd73ac0 100644 --- a/arch/sparc/kernel/prom_irqtrans.c +++ b/arch/sparc/kernel/prom_irqtrans.c @@ -1,8 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/kernel.h> #include <linux/string.h> #include <linux/init.h> #include <linux/of.h> #include <linux/of_platform.h> +#include <linux/platform_device.h> #include <asm/oplib.h> #include <asm/prom.h> @@ -192,7 +194,7 @@ static int sabre_device_needs_wsync(struct device_node *dp) * the DMA synchronization handling */ while (parent) { - if (!strcmp(parent->type, "pci")) + if (of_node_is_type(parent, "pci")) break; parent = parent->parent; } @@ -392,7 +394,7 @@ static unsigned int schizo_irq_build(struct device_node *dp, iclr = schizo_ino_to_iclr(pbm_regs, ino); /* On Schizo, no inofixup occurs. This is because each - * INO has it's own IMAP register. On Psycho and Sabre + * INO has its own IMAP register. On Psycho and Sabre * there is only one IMAP register for each PCI slot even * though four different INOs can be generated by each * PCI slot. @@ -724,11 +726,11 @@ static unsigned int central_build_irq(struct device_node *dp, unsigned long imap, iclr; u32 tmp; - if (!strcmp(dp->name, "eeprom")) { + if (of_node_name_eq(dp, "eeprom")) { res = ¢ral_op->resource[5]; - } else if (!strcmp(dp->name, "zs")) { + } else if (of_node_name_eq(dp, "zs")) { res = ¢ral_op->resource[4]; - } else if (!strcmp(dp->name, "clock-board")) { + } else if (of_node_name_eq(dp, "clock-board")) { res = ¢ral_op->resource[3]; } else { return ino; @@ -823,19 +825,19 @@ void __init irq_trans_init(struct device_node *dp) } #endif #ifdef CONFIG_SBUS - if (!strcmp(dp->name, "sbus") || - !strcmp(dp->name, "sbi")) { + if (of_node_name_eq(dp, "sbus") || + of_node_name_eq(dp, "sbi")) { sbus_irq_trans_init(dp); return; } #endif - if (!strcmp(dp->name, "fhc") && - !strcmp(dp->parent->name, "central")) { + if (of_node_name_eq(dp, "fhc") && + of_node_name_eq(dp->parent, "central")) { central_irq_trans_init(dp); return; } - if (!strcmp(dp->name, "virtual-devices") || - !strcmp(dp->name, "niu")) { + if (of_node_name_eq(dp, "virtual-devices") || + of_node_name_eq(dp, "niu")) { sun4v_vdev_irq_trans_init(dp); return; } diff --git a/arch/sparc/kernel/psycho_common.c b/arch/sparc/kernel/psycho_common.c index 8db48e808ed4..4557ef18f371 100644 --- a/arch/sparc/kernel/psycho_common.c +++ b/arch/sparc/kernel/psycho_common.c @@ -1,9 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 /* psycho_common.c: Code common to PSYCHO and derivative PCI controllers. * * Copyright (C) 2008 David S. Miller <davem@davemloft.net> */ #include <linux/kernel.h> #include <linux/interrupt.h> +#include <linux/numa.h> +#include <linux/platform_device.h> #include <asm/upa.h> @@ -47,7 +50,7 @@ static void psycho_check_stc_error(struct pci_pbm_info *pbm) spin_lock(&stc_buf_lock); /* This is __REALLY__ dangerous. When we put the streaming - * buffer into diagnostic mode to probe it's tags and error + * buffer into diagnostic mode to probe its tags and error * status, we _must_ clear all of the line tag valid bits * before re-enabling the streaming buffer. If any dirty data * lives in the STC when we do this, we will end up @@ -453,7 +456,7 @@ void psycho_pbm_init_common(struct pci_pbm_info *pbm, struct platform_device *op struct device_node *dp = op->dev.of_node; pbm->name = dp->full_name; - pbm->numa_node = -1; + pbm->numa_node = NUMA_NO_NODE; pbm->chip_type = chip_type; pbm->chip_version = of_getintprop_default(dp, "version#", 0); pbm->chip_revision = of_getintprop_default(dp, "module-revision#", 0); diff --git a/arch/sparc/kernel/psycho_common.h b/arch/sparc/kernel/psycho_common.h index 590b4ed8ab5e..6925231c50e4 100644 --- a/arch/sparc/kernel/psycho_common.h +++ b/arch/sparc/kernel/psycho_common.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _PSYCHO_COMMON_H #define _PSYCHO_COMMON_H @@ -30,19 +31,19 @@ enum psycho_error_type { UE_ERR, CE_ERR, PCI_ERR }; -extern void psycho_check_iommu_error(struct pci_pbm_info *pbm, - unsigned long afsr, - unsigned long afar, - enum psycho_error_type type); +void psycho_check_iommu_error(struct pci_pbm_info *pbm, + unsigned long afsr, + unsigned long afar, + enum psycho_error_type type); -extern irqreturn_t psycho_pcierr_intr(int irq, void *dev_id); +irqreturn_t psycho_pcierr_intr(int irq, void *dev_id); -extern int psycho_iommu_init(struct pci_pbm_info *pbm, int tsbsize, - u32 dvma_offset, u32 dma_mask, - unsigned long write_complete_offset); +int psycho_iommu_init(struct pci_pbm_info *pbm, int tsbsize, + u32 dvma_offset, u32 dma_mask, + unsigned long write_complete_offset); -extern void psycho_pbm_init_common(struct pci_pbm_info *pbm, - struct platform_device *op, - const char *chip_name, int chip_type); +void psycho_pbm_init_common(struct pci_pbm_info *pbm, + struct platform_device *op, + const char *chip_name, int chip_type); #endif /* _PSYCHO_COMMON_H */ diff --git a/arch/sparc/kernel/ptrace_32.c b/arch/sparc/kernel/ptrace_32.c index 896ba7c5cd8e..c56333975fb1 100644 --- a/arch/sparc/kernel/ptrace_32.c +++ b/arch/sparc/kernel/ptrace_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* ptrace.c: Sparc process tracing support. * * Copyright (C) 1996, 2008 David S. Miller (davem@davemloft.net) @@ -20,12 +21,12 @@ #include <linux/signal.h> #include <linux/regset.h> #include <linux/elf.h> -#include <linux/tracehook.h> -#include <asm/pgtable.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/cacheflush.h> +#include "kernel.h" + /* #define ALLOW_INIT_TRACING */ /* @@ -43,82 +44,63 @@ enum sparc_regset { REGSET_FP, }; -static int genregs32_get(struct task_struct *target, - const struct user_regset *regset, - unsigned int pos, unsigned int count, - void *kbuf, void __user *ubuf) +static int regwindow32_get(struct task_struct *target, + const struct pt_regs *regs, + u32 *uregs) { - const struct pt_regs *regs = target->thread.kregs; - unsigned long __user *reg_window; - unsigned long *k = kbuf; - unsigned long __user *u = ubuf; - unsigned long reg; - - if (target == current) - flush_user_windows(); + unsigned long reg_window = regs->u_regs[UREG_I6]; + int size = 16 * sizeof(u32); - pos /= sizeof(reg); - count /= sizeof(reg); + if (target == current) { + if (copy_from_user(uregs, (void __user *)reg_window, size)) + return -EFAULT; + } else { + if (access_process_vm(target, reg_window, uregs, size, + FOLL_FORCE) != size) + return -EFAULT; + } + return 0; +} - if (kbuf) { - for (; count > 0 && pos < 16; count--) - *k++ = regs->u_regs[pos++]; +static int regwindow32_set(struct task_struct *target, + const struct pt_regs *regs, + u32 *uregs) +{ + unsigned long reg_window = regs->u_regs[UREG_I6]; + int size = 16 * sizeof(u32); - reg_window = (unsigned long __user *) regs->u_regs[UREG_I6]; - reg_window -= 16; - for (; count > 0 && pos < 32; count--) { - if (get_user(*k++, ®_window[pos++])) - return -EFAULT; - } + if (target == current) { + if (copy_to_user((void __user *)reg_window, uregs, size)) + return -EFAULT; } else { - for (; count > 0 && pos < 16; count--) { - if (put_user(regs->u_regs[pos++], u++)) - return -EFAULT; - } - - reg_window = (unsigned long __user *) regs->u_regs[UREG_I6]; - reg_window -= 16; - for (; count > 0 && pos < 32; count--) { - if (get_user(reg, ®_window[pos++]) || - put_user(reg, u++)) - return -EFAULT; - } - } - while (count > 0) { - switch (pos) { - case 32: /* PSR */ - reg = regs->psr; - break; - case 33: /* PC */ - reg = regs->pc; - break; - case 34: /* NPC */ - reg = regs->npc; - break; - case 35: /* Y */ - reg = regs->y; - break; - case 36: /* WIM */ - case 37: /* TBR */ - reg = 0; - break; - default: - goto finish; - } - - if (kbuf) - *k++ = reg; - else if (put_user(reg, u++)) + if (access_process_vm(target, reg_window, uregs, size, + FOLL_FORCE | FOLL_WRITE) != size) return -EFAULT; - pos++; - count--; } -finish: - pos *= sizeof(reg); - count *= sizeof(reg); + return 0; +} + +static int genregs32_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + const struct pt_regs *regs = target->thread.kregs; + u32 uregs[16]; - return user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, - 38 * sizeof(reg), -1); + if (target == current) + flush_user_windows(); + + membuf_write(&to, regs->u_regs, 16 * sizeof(u32)); + if (!to.left) + return 0; + if (regwindow32_get(target, regs, uregs)) + return -EFAULT; + membuf_write(&to, uregs, 16 * sizeof(u32)); + membuf_store(&to, regs->psr); + membuf_store(&to, regs->pc); + membuf_store(&to, regs->npc); + membuf_store(&to, regs->y); + return membuf_zero(&to, 2 * sizeof(u32)); } static int genregs32_set(struct task_struct *target, @@ -127,126 +109,74 @@ static int genregs32_set(struct task_struct *target, const void *kbuf, const void __user *ubuf) { struct pt_regs *regs = target->thread.kregs; - unsigned long __user *reg_window; - const unsigned long *k = kbuf; - const unsigned long __user *u = ubuf; - unsigned long reg; + u32 uregs[16]; + u32 psr; + int ret; if (target == current) flush_user_windows(); - pos /= sizeof(reg); - count /= sizeof(reg); - - if (kbuf) { - for (; count > 0 && pos < 16; count--) - regs->u_regs[pos++] = *k++; - - reg_window = (unsigned long __user *) regs->u_regs[UREG_I6]; - reg_window -= 16; - for (; count > 0 && pos < 32; count--) { - if (put_user(*k++, ®_window[pos++])) - return -EFAULT; - } - } else { - for (; count > 0 && pos < 16; count--) { - if (get_user(reg, u++)) - return -EFAULT; - regs->u_regs[pos++] = reg; - } - - reg_window = (unsigned long __user *) regs->u_regs[UREG_I6]; - reg_window -= 16; - for (; count > 0 && pos < 32; count--) { - if (get_user(reg, u++) || - put_user(reg, ®_window[pos++])) - return -EFAULT; - } - } - while (count > 0) { - unsigned long psr; - - if (kbuf) - reg = *k++; - else if (get_user(reg, u++)) - return -EFAULT; + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + regs->u_regs, + 0, 16 * sizeof(u32)); + if (ret || !count) + return ret; - switch (pos) { - case 32: /* PSR */ - psr = regs->psr; - psr &= ~(PSR_ICC | PSR_SYSCALL); - psr |= (reg & (PSR_ICC | PSR_SYSCALL)); - regs->psr = psr; - break; - case 33: /* PC */ - regs->pc = reg; - break; - case 34: /* NPC */ - regs->npc = reg; - break; - case 35: /* Y */ - regs->y = reg; - break; - case 36: /* WIM */ - case 37: /* TBR */ - break; - default: - goto finish; - } - - pos++; - count--; - } -finish: - pos *= sizeof(reg); - count *= sizeof(reg); + if (regwindow32_get(target, regs, uregs)) + return -EFAULT; + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + uregs, + 16 * sizeof(u32), 32 * sizeof(u32)); + if (ret) + return ret; + if (regwindow32_set(target, regs, uregs)) + return -EFAULT; + if (!count) + return 0; - return user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, - 38 * sizeof(reg), -1); + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &psr, + 32 * sizeof(u32), 33 * sizeof(u32)); + if (ret) + return ret; + regs->psr = (regs->psr & ~(PSR_ICC | PSR_SYSCALL)) | + (psr & (PSR_ICC | PSR_SYSCALL)); + if (!count) + return 0; + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + ®s->pc, + 33 * sizeof(u32), 34 * sizeof(u32)); + if (ret || !count) + return ret; + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + ®s->npc, + 34 * sizeof(u32), 35 * sizeof(u32)); + if (ret || !count) + return ret; + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + ®s->y, + 35 * sizeof(u32), 36 * sizeof(u32)); + if (ret || !count) + return ret; + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, 36 * sizeof(u32), + 38 * sizeof(u32)); + return 0; } static int fpregs32_get(struct task_struct *target, const struct user_regset *regset, - unsigned int pos, unsigned int count, - void *kbuf, void __user *ubuf) + struct membuf to) { - const unsigned long *fpregs = target->thread.float_regs; - int ret = 0; - #if 0 if (target == current) save_and_clear_fpu(); #endif - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - fpregs, - 0, 32 * sizeof(u32)); - - if (!ret) - ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, - 32 * sizeof(u32), - 33 * sizeof(u32)); - if (!ret) - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - &target->thread.fsr, - 33 * sizeof(u32), - 34 * sizeof(u32)); - - if (!ret) { - unsigned long val; - - val = (1 << 8) | (8 << 16); - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - &val, - 34 * sizeof(u32), - 35 * sizeof(u32)); - } - - if (!ret) - ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, - 35 * sizeof(u32), -1); - - return ret; + membuf_write(&to, target->thread.float_regs, 32 * sizeof(u32)); + membuf_zero(&to, sizeof(u32)); + membuf_write(&to, &target->thread.fsr, sizeof(u32)); + membuf_store(&to, (u32)((1 << 8) | (8 << 16))); + return membuf_zero(&to, 64 * sizeof(u32)); } static int fpregs32_set(struct task_struct *target, @@ -268,16 +198,14 @@ static int fpregs32_set(struct task_struct *target, user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, 32 * sizeof(u32), 33 * sizeof(u32)); - if (!ret && count > 0) { + if (!ret) ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &target->thread.fsr, 33 * sizeof(u32), 34 * sizeof(u32)); - } - if (!ret) - ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, - 34 * sizeof(u32), -1); + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 34 * sizeof(u32), -1); return ret; } @@ -290,10 +218,10 @@ static const struct user_regset sparc32_regsets[] = { * PSR, PC, nPC, Y, WIM, TBR */ [REGSET_GENERAL] = { - .core_note_type = NT_PRSTATUS, + USER_REGSET_NOTE_TYPE(PRSTATUS), .n = 38, .size = sizeof(u32), .align = sizeof(u32), - .get = genregs32_get, .set = genregs32_set + .regset_get = genregs32_get, .set = genregs32_set }, /* Format is: * F0 --> F31 @@ -306,13 +234,107 @@ static const struct user_regset sparc32_regsets[] = { * FPU QUEUE (64 32-bit ints) */ [REGSET_FP] = { - .core_note_type = NT_PRFPREG, + USER_REGSET_NOTE_TYPE(PRFPREG), .n = 99, .size = sizeof(u32), .align = sizeof(u32), - .get = fpregs32_get, .set = fpregs32_set + .regset_get = fpregs32_get, .set = fpregs32_set }, }; +static int getregs_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + const struct pt_regs *regs = target->thread.kregs; + + if (target == current) + flush_user_windows(); + + membuf_store(&to, regs->psr); + membuf_store(&to, regs->pc); + membuf_store(&to, regs->npc); + membuf_store(&to, regs->y); + return membuf_write(&to, regs->u_regs + 1, 15 * sizeof(u32)); +} + +static int setregs_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + struct pt_regs *regs = target->thread.kregs; + u32 v[4]; + int ret; + + if (target == current) + flush_user_windows(); + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + v, + 0, 4 * sizeof(u32)); + if (ret) + return ret; + regs->psr = (regs->psr & ~(PSR_ICC | PSR_SYSCALL)) | + (v[0] & (PSR_ICC | PSR_SYSCALL)); + regs->pc = v[1]; + regs->npc = v[2]; + regs->y = v[3]; + return user_regset_copyin(&pos, &count, &kbuf, &ubuf, + regs->u_regs + 1, + 4 * sizeof(u32) , 19 * sizeof(u32)); +} + +static int getfpregs_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ +#if 0 + if (target == current) + save_and_clear_fpu(); +#endif + membuf_write(&to, &target->thread.float_regs, 32 * sizeof(u32)); + membuf_write(&to, &target->thread.fsr, sizeof(u32)); + return membuf_zero(&to, 35 * sizeof(u32)); +} + +static int setfpregs_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + unsigned long *fpregs = target->thread.float_regs; + int ret; + +#if 0 + if (target == current) + save_and_clear_fpu(); +#endif + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + fpregs, + 0, 32 * sizeof(u32)); + if (ret) + return ret; + return user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fsr, + 32 * sizeof(u32), + 33 * sizeof(u32)); +} + +static const struct user_regset ptrace32_regsets[] = { + [REGSET_GENERAL] = { + .n = 19, .size = sizeof(u32), + .regset_get = getregs_get, .set = setregs_set, + }, + [REGSET_FP] = { + .n = 68, .size = sizeof(u32), + .regset_get = getfpregs_get, .set = setfpregs_set, + }, +}; + +static const struct user_regset_view ptrace32_view = { + .regsets = ptrace32_regsets, .n = ARRAY_SIZE(ptrace32_regsets) +}; + static const struct user_regset_view user_sparc32_view = { .name = "sparc", .e_machine = EM_SPARC, .regsets = sparc32_regsets, .n = ARRAY_SIZE(sparc32_regsets) @@ -340,74 +362,44 @@ long arch_ptrace(struct task_struct *child, long request, { unsigned long addr2 = current->thread.kregs->u_regs[UREG_I4]; void __user *addr2p; - const struct user_regset_view *view; struct pt_regs __user *pregs; struct fps __user *fps; int ret; - view = task_user_regset_view(current); addr2p = (void __user *) addr2; pregs = (struct pt_regs __user *) addr; fps = (struct fps __user *) addr; switch(request) { case PTRACE_GETREGS: { - ret = copy_regset_to_user(child, view, REGSET_GENERAL, - 32 * sizeof(u32), - 4 * sizeof(u32), - &pregs->psr); - if (!ret) - copy_regset_to_user(child, view, REGSET_GENERAL, - 1 * sizeof(u32), - 15 * sizeof(u32), - &pregs->u_regs[0]); + ret = copy_regset_to_user(child, &ptrace32_view, + REGSET_GENERAL, 0, + 19 * sizeof(u32), + pregs); break; } case PTRACE_SETREGS: { - ret = copy_regset_from_user(child, view, REGSET_GENERAL, - 32 * sizeof(u32), - 4 * sizeof(u32), - &pregs->psr); - if (!ret) - copy_regset_from_user(child, view, REGSET_GENERAL, - 1 * sizeof(u32), - 15 * sizeof(u32), - &pregs->u_regs[0]); + ret = copy_regset_from_user(child, &ptrace32_view, + REGSET_GENERAL, 0, + 19 * sizeof(u32), + pregs); break; } case PTRACE_GETFPREGS: { - ret = copy_regset_to_user(child, view, REGSET_FP, - 0 * sizeof(u32), - 32 * sizeof(u32), - &fps->regs[0]); - if (!ret) - ret = copy_regset_to_user(child, view, REGSET_FP, - 33 * sizeof(u32), - 1 * sizeof(u32), - &fps->fsr); - - if (!ret) { - if (__put_user(0, &fps->fpqd) || - __put_user(0, &fps->flags) || - __put_user(0, &fps->extra) || - clear_user(fps->fpq, sizeof(fps->fpq))) - ret = -EFAULT; - } + ret = copy_regset_to_user(child, &ptrace32_view, + REGSET_FP, 0, + 68 * sizeof(u32), + fps); break; } case PTRACE_SETFPREGS: { - ret = copy_regset_from_user(child, view, REGSET_FP, - 0 * sizeof(u32), - 32 * sizeof(u32), - &fps->regs[0]); - if (!ret) - ret = copy_regset_from_user(child, view, REGSET_FP, - 33 * sizeof(u32), - 1 * sizeof(u32), - &fps->fsr); + ret = copy_regset_from_user(child, &ptrace32_view, + REGSET_FP, 0, + 33 * sizeof(u32), + fps); break; } @@ -447,9 +439,9 @@ asmlinkage int syscall_trace(struct pt_regs *regs, int syscall_exit_p) if (test_thread_flag(TIF_SYSCALL_TRACE)) { if (syscall_exit_p) - tracehook_report_syscall_exit(regs, 0); + ptrace_report_syscall_exit(regs, 0); else - ret = tracehook_report_syscall_entry(regs); + ret = ptrace_report_syscall_entry(regs); } return ret; diff --git a/arch/sparc/kernel/ptrace_64.c b/arch/sparc/kernel/ptrace_64.c index 7ff45e4ba681..9fc67fa9336f 100644 --- a/arch/sparc/kernel/ptrace_64.c +++ b/arch/sparc/kernel/ptrace_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* ptrace.c: Sparc process tracing support. * * Copyright (C) 1996, 2008 David S. Miller (davem@davemloft.net) @@ -12,8 +13,10 @@ #include <linux/kernel.h> #include <linux/sched.h> +#include <linux/sched/task_stack.h> #include <linux/mm.h> #include <linux/errno.h> +#include <linux/export.h> #include <linux/ptrace.h> #include <linux/user.h> #include <linux/smp.h> @@ -22,14 +25,13 @@ #include <linux/audit.h> #include <linux/signal.h> #include <linux/regset.h> -#include <linux/tracehook.h> #include <trace/syscall.h> #include <linux/compat.h> #include <linux/elf.h> +#include <linux/context_tracking.h> #include <asm/asi.h> -#include <asm/pgtable.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/psrcompat.h> #include <asm/visasm.h> #include <asm/spitfire.h> @@ -44,6 +46,43 @@ /* #define ALLOW_INIT_TRACING */ +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define REG_OFFSET_NAME(n, r) \ + {.name = n, .offset = (PT_V9_##r)} +#define REG_OFFSET_END {.name = NULL, .offset = 0} + +static const struct pt_regs_offset regoffset_table[] = { + REG_OFFSET_NAME("g0", G0), + REG_OFFSET_NAME("g1", G1), + REG_OFFSET_NAME("g2", G2), + REG_OFFSET_NAME("g3", G3), + REG_OFFSET_NAME("g4", G4), + REG_OFFSET_NAME("g5", G5), + REG_OFFSET_NAME("g6", G6), + REG_OFFSET_NAME("g7", G7), + + REG_OFFSET_NAME("i0", I0), + REG_OFFSET_NAME("i1", I1), + REG_OFFSET_NAME("i2", I2), + REG_OFFSET_NAME("i3", I3), + REG_OFFSET_NAME("i4", I4), + REG_OFFSET_NAME("i5", I5), + REG_OFFSET_NAME("i6", I6), + REG_OFFSET_NAME("i7", I7), + + REG_OFFSET_NAME("tstate", TSTATE), + REG_OFFSET_NAME("pc", TPC), + REG_OFFSET_NAME("npc", TNPC), + REG_OFFSET_NAME("y", Y), + REG_OFFSET_NAME("lr", I7), + + REG_OFFSET_END, +}; + /* * Called by kernel/ptrace.c when detaching.. * @@ -116,6 +155,7 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, preempt_enable(); } +EXPORT_SYMBOL_GPL(flush_ptrace_access); static int get_from_target(struct task_struct *target, unsigned long uaddr, void *kbuf, int len) @@ -124,7 +164,8 @@ static int get_from_target(struct task_struct *target, unsigned long uaddr, if (copy_from_user(kbuf, (void __user *) uaddr, len)) return -EFAULT; } else { - int len2 = access_process_vm(target, uaddr, kbuf, len, 0); + int len2 = access_process_vm(target, uaddr, kbuf, len, + FOLL_FORCE); if (len2 != len) return -EFAULT; } @@ -138,7 +179,8 @@ static int set_to_target(struct task_struct *target, unsigned long uaddr, if (copy_to_user((void __user *) uaddr, kbuf, len)) return -EFAULT; } else { - int len2 = access_process_vm(target, uaddr, kbuf, len, 1); + int len2 = access_process_vm(target, uaddr, kbuf, len, + FOLL_FORCE | FOLL_WRITE); if (len2 != len) return -EFAULT; } @@ -203,52 +245,23 @@ enum sparc_regset { static int genregs64_get(struct task_struct *target, const struct user_regset *regset, - unsigned int pos, unsigned int count, - void *kbuf, void __user *ubuf) + struct membuf to) { const struct pt_regs *regs = task_pt_regs(target); - int ret; + struct reg_window window; if (target == current) flushw_user(); - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - regs->u_regs, - 0, 16 * sizeof(u64)); - if (!ret && count && pos < (32 * sizeof(u64))) { - struct reg_window window; - - if (regwindow64_get(target, regs, &window)) - return -EFAULT; - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - &window, - 16 * sizeof(u64), - 32 * sizeof(u64)); - } - - if (!ret) { - /* TSTATE, TPC, TNPC */ - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - ®s->tstate, - 32 * sizeof(u64), - 35 * sizeof(u64)); - } - - if (!ret) { - unsigned long y = regs->y; - - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - &y, - 35 * sizeof(u64), - 36 * sizeof(u64)); - } - - if (!ret) { - ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, - 36 * sizeof(u64), -1); - - } - return ret; + membuf_write(&to, regs->u_regs, 16 * sizeof(u64)); + if (!to.left) + return 0; + if (regwindow64_get(target, regs, &window)) + return -EFAULT; + membuf_write(&to, &window, 16 * sizeof(u64)); + /* TSTATE, TPC, TNPC */ + membuf_write(&to, ®s->tstate, 3 * sizeof(u64)); + return membuf_store(&to, (u64)regs->y); } static int genregs64_set(struct task_struct *target, @@ -308,7 +321,7 @@ static int genregs64_set(struct task_struct *target, } if (!ret) { - unsigned long y; + unsigned long y = regs->y; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &y, @@ -319,77 +332,40 @@ static int genregs64_set(struct task_struct *target, } if (!ret) - ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, - 36 * sizeof(u64), -1); + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 36 * sizeof(u64), -1); return ret; } static int fpregs64_get(struct task_struct *target, const struct user_regset *regset, - unsigned int pos, unsigned int count, - void *kbuf, void __user *ubuf) + struct membuf to) { - const unsigned long *fpregs = task_thread_info(target)->fpregs; - unsigned long fprs, fsr, gsr; - int ret; + struct thread_info *t = task_thread_info(target); + unsigned long fprs; if (target == current) save_and_clear_fpu(); - fprs = task_thread_info(target)->fpsaved[0]; + fprs = t->fpsaved[0]; if (fprs & FPRS_DL) - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - fpregs, - 0, 16 * sizeof(u64)); + membuf_write(&to, t->fpregs, 16 * sizeof(u64)); else - ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, - 0, - 16 * sizeof(u64)); - - if (!ret) { - if (fprs & FPRS_DU) - ret = user_regset_copyout(&pos, &count, - &kbuf, &ubuf, - fpregs + 16, - 16 * sizeof(u64), - 32 * sizeof(u64)); - else - ret = user_regset_copyout_zero(&pos, &count, - &kbuf, &ubuf, - 16 * sizeof(u64), - 32 * sizeof(u64)); - } + membuf_zero(&to, 16 * sizeof(u64)); + if (fprs & FPRS_DU) + membuf_write(&to, t->fpregs + 16, 16 * sizeof(u64)); + else + membuf_zero(&to, 16 * sizeof(u64)); if (fprs & FPRS_FEF) { - fsr = task_thread_info(target)->xfsr[0]; - gsr = task_thread_info(target)->gsr[0]; + membuf_store(&to, t->xfsr[0]); + membuf_store(&to, t->gsr[0]); } else { - fsr = gsr = 0; + membuf_zero(&to, 2 * sizeof(u64)); } - - if (!ret) - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - &fsr, - 32 * sizeof(u64), - 33 * sizeof(u64)); - if (!ret) - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - &gsr, - 33 * sizeof(u64), - 34 * sizeof(u64)); - if (!ret) - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - &fprs, - 34 * sizeof(u64), - 35 * sizeof(u64)); - - if (!ret) - ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, - 35 * sizeof(u64), -1); - - return ret; + return membuf_store(&to, fprs); } static int fpregs64_set(struct task_struct *target, @@ -430,8 +406,8 @@ static int fpregs64_set(struct task_struct *target, task_thread_info(target)->fpsaved[0] = fprs; if (!ret) - ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, - 35 * sizeof(u64), -1); + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 35 * sizeof(u64), -1); return ret; } @@ -444,10 +420,10 @@ static const struct user_regset sparc64_regsets[] = { * TSTATE, TPC, TNPC, Y */ [REGSET_GENERAL] = { - .core_note_type = NT_PRSTATUS, + USER_REGSET_NOTE_TYPE(PRSTATUS), .n = 36, .size = sizeof(u64), .align = sizeof(u64), - .get = genregs64_get, .set = genregs64_set + .regset_get = genregs64_get, .set = genregs64_set }, /* Format is: * F0 --> F63 @@ -456,13 +432,97 @@ static const struct user_regset sparc64_regsets[] = { * FPRS */ [REGSET_FP] = { - .core_note_type = NT_PRFPREG, + USER_REGSET_NOTE_TYPE(PRFPREG), .n = 35, .size = sizeof(u64), .align = sizeof(u64), - .get = fpregs64_get, .set = fpregs64_set + .regset_get = fpregs64_get, .set = fpregs64_set }, }; +static int getregs64_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + const struct pt_regs *regs = task_pt_regs(target); + + if (target == current) + flushw_user(); + + membuf_write(&to, regs->u_regs + 1, 15 * sizeof(u64)); + membuf_store(&to, (u64)0); + membuf_write(&to, ®s->tstate, 3 * sizeof(u64)); + return membuf_store(&to, (u64)regs->y); +} + +static int setregs64_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + struct pt_regs *regs = task_pt_regs(target); + unsigned long y = regs->y; + unsigned long tstate; + int ret; + + if (target == current) + flushw_user(); + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + regs->u_regs + 1, + 0 * sizeof(u64), + 15 * sizeof(u64)); + if (ret) + return ret; + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 15 * sizeof(u64), 16 * sizeof(u64)); + /* TSTATE */ + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &tstate, + 16 * sizeof(u64), + 17 * sizeof(u64)); + if (ret) + return ret; + /* Only the condition codes and the "in syscall" + * state can be modified in the %tstate register. + */ + tstate &= (TSTATE_ICC | TSTATE_XCC | TSTATE_SYSCALL); + regs->tstate &= ~(TSTATE_ICC | TSTATE_XCC | TSTATE_SYSCALL); + regs->tstate |= tstate; + + /* TPC, TNPC */ + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + ®s->tpc, + 17 * sizeof(u64), + 19 * sizeof(u64)); + if (ret) + return ret; + /* Y */ + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &y, + 19 * sizeof(u64), + 20 * sizeof(u64)); + if (!ret) + regs->y = y; + return ret; +} + +static const struct user_regset ptrace64_regsets[] = { + /* Format is: + * G1 --> G7 + * O0 --> O7 + * 0 + * TSTATE, TPC, TNPC, Y + */ + [REGSET_GENERAL] = { + .n = 20, .size = sizeof(u64), + .regset_get = getregs64_get, .set = setregs64_set, + }, +}; + +static const struct user_regset_view ptrace64_view = { + .regsets = ptrace64_regsets, .n = ARRAY_SIZE(ptrace64_regsets) +}; + static const struct user_regset_view user_sparc64_view = { .name = "sparc64", .e_machine = EM_SPARCV9, .regsets = sparc64_regsets, .n = ARRAY_SIZE(sparc64_regsets) @@ -471,111 +531,28 @@ static const struct user_regset_view user_sparc64_view = { #ifdef CONFIG_COMPAT static int genregs32_get(struct task_struct *target, const struct user_regset *regset, - unsigned int pos, unsigned int count, - void *kbuf, void __user *ubuf) + struct membuf to) { const struct pt_regs *regs = task_pt_regs(target); - compat_ulong_t __user *reg_window; - compat_ulong_t *k = kbuf; - compat_ulong_t __user *u = ubuf; - compat_ulong_t reg; + u32 uregs[16]; + int i; if (target == current) flushw_user(); - pos /= sizeof(reg); - count /= sizeof(reg); - - if (kbuf) { - for (; count > 0 && pos < 16; count--) - *k++ = regs->u_regs[pos++]; - - reg_window = (compat_ulong_t __user *) regs->u_regs[UREG_I6]; - reg_window -= 16; - if (target == current) { - for (; count > 0 && pos < 32; count--) { - if (get_user(*k++, ®_window[pos++])) - return -EFAULT; - } - } else { - for (; count > 0 && pos < 32; count--) { - if (access_process_vm(target, - (unsigned long) - ®_window[pos], - k, sizeof(*k), 0) - != sizeof(*k)) - return -EFAULT; - k++; - pos++; - } - } - } else { - for (; count > 0 && pos < 16; count--) { - if (put_user((compat_ulong_t) regs->u_regs[pos++], u++)) - return -EFAULT; - } - - reg_window = (compat_ulong_t __user *) regs->u_regs[UREG_I6]; - reg_window -= 16; - if (target == current) { - for (; count > 0 && pos < 32; count--) { - if (get_user(reg, ®_window[pos++]) || - put_user(reg, u++)) - return -EFAULT; - } - } else { - for (; count > 0 && pos < 32; count--) { - if (access_process_vm(target, - (unsigned long) - ®_window[pos], - ®, sizeof(reg), 0) - != sizeof(reg)) - return -EFAULT; - if (access_process_vm(target, - (unsigned long) u, - ®, sizeof(reg), 1) - != sizeof(reg)) - return -EFAULT; - pos++; - u++; - } - } - } - while (count > 0) { - switch (pos) { - case 32: /* PSR */ - reg = tstate_to_psr(regs->tstate); - break; - case 33: /* PC */ - reg = regs->tpc; - break; - case 34: /* NPC */ - reg = regs->tnpc; - break; - case 35: /* Y */ - reg = regs->y; - break; - case 36: /* WIM */ - case 37: /* TBR */ - reg = 0; - break; - default: - goto finish; - } - - if (kbuf) - *k++ = reg; - else if (put_user(reg, u++)) - return -EFAULT; - pos++; - count--; - } -finish: - pos *= sizeof(reg); - count *= sizeof(reg); - - return user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, - 38 * sizeof(reg), -1); + for (i = 0; i < 16; i++) + membuf_store(&to, (u32)regs->u_regs[i]); + if (!to.left) + return 0; + if (get_from_target(target, regs->u_regs[UREG_I6], + uregs, sizeof(uregs))) + return -EFAULT; + membuf_write(&to, uregs, 16 * sizeof(u32)); + membuf_store(&to, (u32)tstate_to_psr(regs->tstate)); + membuf_store(&to, (u32)(regs->tpc)); + membuf_store(&to, (u32)(regs->tnpc)); + membuf_store(&to, (u32)(regs->y)); + return membuf_zero(&to, 2 * sizeof(u32)); } static int genregs32_set(struct task_struct *target, @@ -612,7 +589,8 @@ static int genregs32_set(struct task_struct *target, (unsigned long) ®_window[pos], (void *) k, - sizeof(*k), 1) + sizeof(*k), + FOLL_FORCE | FOLL_WRITE) != sizeof(*k)) return -EFAULT; k++; @@ -636,16 +614,13 @@ static int genregs32_set(struct task_struct *target, } } else { for (; count > 0 && pos < 32; count--) { - if (access_process_vm(target, - (unsigned long) - u, - ®, sizeof(reg), 0) - != sizeof(reg)) + if (get_user(reg, u++)) return -EFAULT; if (access_process_vm(target, (unsigned long) ®_window[pos], - ®, sizeof(reg), 1) + ®, sizeof(reg), + FOLL_FORCE | FOLL_WRITE) != sizeof(reg)) return -EFAULT; pos++; @@ -693,62 +668,31 @@ finish: pos *= sizeof(reg); count *= sizeof(reg); - return user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, - 38 * sizeof(reg), -1); + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 38 * sizeof(reg), -1); + return 0; } static int fpregs32_get(struct task_struct *target, const struct user_regset *regset, - unsigned int pos, unsigned int count, - void *kbuf, void __user *ubuf) + struct membuf to) { - const unsigned long *fpregs = task_thread_info(target)->fpregs; - compat_ulong_t enabled; - unsigned long fprs; - compat_ulong_t fsr; - int ret = 0; + struct thread_info *t = task_thread_info(target); + bool enabled; if (target == current) save_and_clear_fpu(); - fprs = task_thread_info(target)->fpsaved[0]; - if (fprs & FPRS_FEF) { - fsr = task_thread_info(target)->xfsr[0]; - enabled = 1; - } else { - fsr = 0; - enabled = 0; - } - - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - fpregs, - 0, 32 * sizeof(u32)); - - if (!ret) - ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, - 32 * sizeof(u32), - 33 * sizeof(u32)); - if (!ret) - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - &fsr, - 33 * sizeof(u32), - 34 * sizeof(u32)); - - if (!ret) { - compat_ulong_t val; - - val = (enabled << 8) | (8 << 16); - ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, - &val, - 34 * sizeof(u32), - 35 * sizeof(u32)); - } - - if (!ret) - ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, - 35 * sizeof(u32), -1); + enabled = t->fpsaved[0] & FPRS_FEF; - return ret; + membuf_write(&to, t->fpregs, 32 * sizeof(u32)); + membuf_zero(&to, sizeof(u32)); + if (enabled) + membuf_store(&to, (u32)t->xfsr[0]); + else + membuf_zero(&to, sizeof(u32)); + membuf_store(&to, (u32)((enabled << 8) | (8 << 16))); + return membuf_zero(&to, 64 * sizeof(u32)); } static int fpregs32_set(struct task_struct *target, @@ -792,8 +736,8 @@ static int fpregs32_set(struct task_struct *target, task_thread_info(target)->fpsaved[0] = fprs; if (!ret) - ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, - 34 * sizeof(u32), -1); + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 34 * sizeof(u32), -1); return ret; } @@ -806,10 +750,10 @@ static const struct user_regset sparc32_regsets[] = { * PSR, PC, nPC, Y, WIM, TBR */ [REGSET_GENERAL] = { - .core_note_type = NT_PRSTATUS, + USER_REGSET_NOTE_TYPE(PRSTATUS), .n = 38, .size = sizeof(u32), .align = sizeof(u32), - .get = genregs32_get, .set = genregs32_set + .regset_get = genregs32_get, .set = genregs32_set }, /* Format is: * F0 --> F31 @@ -822,11 +766,134 @@ static const struct user_regset sparc32_regsets[] = { * FPU QUEUE (64 32-bit ints) */ [REGSET_FP] = { - .core_note_type = NT_PRFPREG, + USER_REGSET_NOTE_TYPE(PRFPREG), .n = 99, .size = sizeof(u32), .align = sizeof(u32), - .get = fpregs32_get, .set = fpregs32_set + .regset_get = fpregs32_get, .set = fpregs32_set + }, +}; + +static int getregs_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + const struct pt_regs *regs = task_pt_regs(target); + int i; + + if (target == current) + flushw_user(); + + membuf_store(&to, (u32)tstate_to_psr(regs->tstate)); + membuf_store(&to, (u32)(regs->tpc)); + membuf_store(&to, (u32)(regs->tnpc)); + membuf_store(&to, (u32)(regs->y)); + for (i = 1; i < 16; i++) + membuf_store(&to, (u32)regs->u_regs[i]); + return to.left; +} + +static int setregs_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + struct pt_regs *regs = task_pt_regs(target); + unsigned long tstate; + u32 uregs[19]; + int i, ret; + + if (target == current) + flushw_user(); + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + uregs, + 0, 19 * sizeof(u32)); + if (ret) + return ret; + + tstate = regs->tstate; + tstate &= ~(TSTATE_ICC | TSTATE_XCC | TSTATE_SYSCALL); + tstate |= psr_to_tstate_icc(uregs[0]); + if (uregs[0] & PSR_SYSCALL) + tstate |= TSTATE_SYSCALL; + regs->tstate = tstate; + regs->tpc = uregs[1]; + regs->tnpc = uregs[2]; + regs->y = uregs[3]; + + for (i = 1; i < 15; i++) + regs->u_regs[i] = uregs[3 + i]; + return 0; +} + +static int getfpregs_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + struct thread_info *t = task_thread_info(target); + + if (target == current) + save_and_clear_fpu(); + + membuf_write(&to, t->fpregs, 32 * sizeof(u32)); + if (t->fpsaved[0] & FPRS_FEF) + membuf_store(&to, (u32)t->xfsr[0]); + else + membuf_zero(&to, sizeof(u32)); + return membuf_zero(&to, 35 * sizeof(u32)); +} + +static int setfpregs_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + unsigned long *fpregs = task_thread_info(target)->fpregs; + unsigned long fprs; + int ret; + + if (target == current) + save_and_clear_fpu(); + + fprs = task_thread_info(target)->fpsaved[0]; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + fpregs, + 0, 32 * sizeof(u32)); + if (!ret) { + compat_ulong_t fsr; + unsigned long val; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &fsr, + 32 * sizeof(u32), + 33 * sizeof(u32)); + if (!ret) { + val = task_thread_info(target)->xfsr[0]; + val &= 0xffffffff00000000UL; + val |= fsr; + task_thread_info(target)->xfsr[0] = val; + } + } + + fprs |= (FPRS_FEF | FPRS_DL); + task_thread_info(target)->fpsaved[0] = fprs; + return ret; +} + +static const struct user_regset ptrace32_regsets[] = { + [REGSET_GENERAL] = { + .n = 19, .size = sizeof(u32), + .regset_get = getregs_get, .set = setregs_set, }, + [REGSET_FP] = { + .n = 68, .size = sizeof(u32), + .regset_get = getfpregs_get, .set = setfpregs_set, + }, +}; + +static const struct user_regset_view ptrace32_view = { + .regsets = ptrace32_regsets, .n = ARRAY_SIZE(ptrace32_regsets) }; static const struct user_regset_view user_sparc32_view = { @@ -860,7 +927,6 @@ struct compat_fps { long compat_arch_ptrace(struct task_struct *child, compat_long_t request, compat_ulong_t caddr, compat_ulong_t cdata) { - const struct user_regset_view *view = task_user_regset_view(current); compat_ulong_t caddr2 = task_pt_regs(current)->u_regs[UREG_I4]; struct pt_regs32 __user *pregs; struct compat_fps __user *fps; @@ -878,58 +944,31 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, break; case PTRACE_GETREGS: - ret = copy_regset_to_user(child, view, REGSET_GENERAL, - 32 * sizeof(u32), - 4 * sizeof(u32), - &pregs->psr); - if (!ret) - ret = copy_regset_to_user(child, view, REGSET_GENERAL, - 1 * sizeof(u32), - 15 * sizeof(u32), - &pregs->u_regs[0]); + ret = copy_regset_to_user(child, &ptrace32_view, + REGSET_GENERAL, 0, + 19 * sizeof(u32), + pregs); break; case PTRACE_SETREGS: - ret = copy_regset_from_user(child, view, REGSET_GENERAL, - 32 * sizeof(u32), - 4 * sizeof(u32), - &pregs->psr); - if (!ret) - ret = copy_regset_from_user(child, view, REGSET_GENERAL, - 1 * sizeof(u32), - 15 * sizeof(u32), - &pregs->u_regs[0]); + ret = copy_regset_from_user(child, &ptrace32_view, + REGSET_GENERAL, 0, + 19 * sizeof(u32), + pregs); break; case PTRACE_GETFPREGS: - ret = copy_regset_to_user(child, view, REGSET_FP, - 0 * sizeof(u32), - 32 * sizeof(u32), - &fps->regs[0]); - if (!ret) - ret = copy_regset_to_user(child, view, REGSET_FP, - 33 * sizeof(u32), - 1 * sizeof(u32), - &fps->fsr); - if (!ret) { - if (__put_user(0, &fps->flags) || - __put_user(0, &fps->extra) || - __put_user(0, &fps->fpqd) || - clear_user(&fps->fpq[0], 32 * sizeof(unsigned int))) - ret = -EFAULT; - } + ret = copy_regset_to_user(child, &ptrace32_view, + REGSET_FP, 0, + 68 * sizeof(u32), + fps); break; case PTRACE_SETFPREGS: - ret = copy_regset_from_user(child, view, REGSET_FP, - 0 * sizeof(u32), - 32 * sizeof(u32), - &fps->regs[0]); - if (!ret) - ret = copy_regset_from_user(child, view, REGSET_FP, - 33 * sizeof(u32), - 1 * sizeof(u32), - &fps->fsr); + ret = copy_regset_from_user(child, &ptrace32_view, + REGSET_FP, 0, + 33 * sizeof(u32), + fps); break; case PTRACE_READTEXT: @@ -988,31 +1027,17 @@ long arch_ptrace(struct task_struct *child, long request, break; case PTRACE_GETREGS64: - ret = copy_regset_to_user(child, view, REGSET_GENERAL, - 1 * sizeof(u64), - 15 * sizeof(u64), - &pregs->u_regs[0]); - if (!ret) { - /* XXX doesn't handle 'y' register correctly XXX */ - ret = copy_regset_to_user(child, view, REGSET_GENERAL, - 32 * sizeof(u64), - 4 * sizeof(u64), - &pregs->tstate); - } + ret = copy_regset_to_user(child, &ptrace64_view, + REGSET_GENERAL, 0, + 19 * sizeof(u64), + pregs); break; case PTRACE_SETREGS64: - ret = copy_regset_from_user(child, view, REGSET_GENERAL, - 1 * sizeof(u64), - 15 * sizeof(u64), - &pregs->u_regs[0]); - if (!ret) { - /* XXX doesn't handle 'y' register correctly XXX */ - ret = copy_regset_from_user(child, view, REGSET_GENERAL, - 32 * sizeof(u64), - 4 * sizeof(u64), - &pregs->tstate); - } + ret = copy_regset_from_user(child, &ptrace64_view, + REGSET_GENERAL, 0, + 19 * sizeof(u64), + pregs); break; case PTRACE_GETFPREGS64: @@ -1064,19 +1089,17 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs) /* do the secure computing check first */ secure_computing_strict(regs->u_regs[UREG_G1]); + if (test_thread_flag(TIF_NOHZ)) + user_exit(); + if (test_thread_flag(TIF_SYSCALL_TRACE)) - ret = tracehook_report_syscall_entry(regs); + ret = ptrace_report_syscall_entry(regs); if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) trace_sys_enter(regs, regs->u_regs[UREG_G1]); - audit_syscall_entry((test_thread_flag(TIF_32BIT) ? - AUDIT_ARCH_SPARC : - AUDIT_ARCH_SPARC64), - regs->u_regs[UREG_G1], - regs->u_regs[UREG_I0], - regs->u_regs[UREG_I1], - regs->u_regs[UREG_I2], + audit_syscall_entry(regs->u_regs[UREG_G1], regs->u_regs[UREG_I0], + regs->u_regs[UREG_I1], regs->u_regs[UREG_I2], regs->u_regs[UREG_I3]); return ret; @@ -1084,11 +1107,70 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs) asmlinkage void syscall_trace_leave(struct pt_regs *regs) { + if (test_thread_flag(TIF_NOHZ)) + user_exit(); + audit_syscall_exit(regs); if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) - trace_sys_exit(regs, regs->u_regs[UREG_G1]); + trace_sys_exit(regs, regs->u_regs[UREG_I0]); if (test_thread_flag(TIF_SYSCALL_TRACE)) - tracehook_report_syscall_exit(regs, 0); + ptrace_report_syscall_exit(regs, 0); + + if (test_thread_flag(TIF_NOHZ)) + user_enter(); +} + +/** + * regs_query_register_offset() - query register offset from its name + * @name: the name of a register + * + * regs_query_register_offset() returns the offset of a register in struct + * pt_regs from its name. If the name is invalid, this returns -EINVAL; + */ +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + return -EINVAL; +} + +/** + * regs_within_kernel_stack() - check the address in the stack + * @regs: pt_regs which contains kernel stack pointer. + * @addr: address which is checked. + * + * regs_within_kernel_stack() checks @addr is within the kernel stack page(s). + * If @addr is within the kernel stack, it returns true. If not, returns false. + */ +static inline int regs_within_kernel_stack(struct pt_regs *regs, + unsigned long addr) +{ + unsigned long ksp = kernel_stack_pointer(regs) + STACK_BIAS; + return ((addr & ~(THREAD_SIZE - 1)) == + (ksp & ~(THREAD_SIZE - 1))); +} + +/** + * regs_get_kernel_stack_nth() - get Nth entry of the stack + * @regs: pt_regs which contains kernel stack pointer. + * @n: stack entry number. + * + * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which + * is specified by @regs. If the @n th entry is NOT in the kernel stack, + * this returns 0. + */ +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) +{ + unsigned long ksp = kernel_stack_pointer(regs) + STACK_BIAS; + unsigned long *addr = (unsigned long *)ksp; + addr += n; + if (regs_within_kernel_stack(regs, (unsigned long)addr)) + return *addr; + else + return 0; } diff --git a/arch/sparc/kernel/reboot.c b/arch/sparc/kernel/reboot.c index eba7d918162a..69c1b6c047d5 100644 --- a/arch/sparc/kernel/reboot.c +++ b/arch/sparc/kernel/reboot.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* reboot.c: reboot/shutdown/halt/poweroff handling * * Copyright (C) 2008 David S. Miller <davem@davemloft.net> @@ -6,6 +7,7 @@ #include <linux/reboot.h> #include <linux/export.h> #include <linux/pm.h> +#include <linux/of.h> #include <asm/oplib.h> #include <asm/prom.h> @@ -24,7 +26,7 @@ EXPORT_SYMBOL(pm_power_off); void machine_power_off(void) { - if (strcmp(of_console_device->type, "serial") || scons_pwroff) + if (!of_node_is_type(of_console_device, "serial") || scons_pwroff) prom_halt_power_off(); prom_halt(); diff --git a/arch/sparc/kernel/rtrap_32.S b/arch/sparc/kernel/rtrap_32.S index 6c34de0c2abd..8931fe266346 100644 --- a/arch/sparc/kernel/rtrap_32.S +++ b/arch/sparc/kernel/rtrap_32.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * rtrap.S: Return from Sparc trap low-level code. * @@ -74,7 +75,7 @@ signal_p: ld [%sp + STACKFRAME_SZ + PT_PSR], %t_psr mov %g2, %o2 - mov %l5, %o1 + mov %l6, %o1 call do_notify_resume add %sp, STACKFRAME_SZ, %o0 ! pt_regs ptr diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S index afa2a9e3d0a0..eef102765a7e 100644 --- a/arch/sparc/kernel/rtrap_64.S +++ b/arch/sparc/kernel/rtrap_64.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * rtrap.S: Preparing for return from trap on Sparc V9. * @@ -14,21 +15,42 @@ #include <asm/visasm.h> #include <asm/processor.h> -#define RTRAP_PSTATE (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_IE) -#define RTRAP_PSTATE_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV) -#define RTRAP_PSTATE_AG_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_AG) +#ifdef CONFIG_CONTEXT_TRACKING_USER +# define SCHEDULE_USER schedule_user +#else +# define SCHEDULE_USER schedule +#endif .text .align 32 __handle_preemption: - call schedule - wrpr %g0, RTRAP_PSTATE, %pstate + call SCHEDULE_USER +661: wrpr %g0, RTRAP_PSTATE, %pstate + /* If userspace is using ADI, it could potentially pass + * a pointer with version tag embedded in it. To maintain + * the ADI security, we must re-enable PSTATE.mcde before + * we continue execution in the kernel for another thread. + */ + .section .sun_m7_1insn_patch, "ax" + .word 661b + wrpr %g0, RTRAP_PSTATE|PSTATE_MCDE, %pstate + .previous ba,pt %xcc, __handle_preemption_continue wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate __handle_user_windows: + add %sp, PTREGS_OFF, %o0 call fault_in_user_windows - wrpr %g0, RTRAP_PSTATE, %pstate +661: wrpr %g0, RTRAP_PSTATE, %pstate + /* If userspace is using ADI, it could potentially pass + * a pointer with version tag embedded in it. To maintain + * the ADI security, we must re-enable PSTATE.mcde before + * we continue execution in the kernel for another thread. + */ + .section .sun_m7_1insn_patch, "ax" + .word 661b + wrpr %g0, RTRAP_PSTATE|PSTATE_MCDE, %pstate + .previous ba,pt %xcc, __handle_preemption_continue wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate @@ -45,7 +67,16 @@ __handle_signal: add %sp, PTREGS_OFF, %o0 mov %l0, %o2 call do_notify_resume - wrpr %g0, RTRAP_PSTATE, %pstate +661: wrpr %g0, RTRAP_PSTATE, %pstate + /* If userspace is using ADI, it could potentially pass + * a pointer with version tag embedded in it. To maintain + * the ADI security, we must re-enable PSTATE.mcde before + * we continue execution in the kernel for another thread. + */ + .section .sun_m7_1insn_patch, "ax" + .word 661b + wrpr %g0, RTRAP_PSTATE|PSTATE_MCDE, %pstate + .previous wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate /* Signal delivery can modify pt_regs tstate, so we must @@ -54,8 +85,9 @@ __handle_signal: ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 sethi %hi(0xf << 20), %l4 and %l1, %l4, %l4 + andn %l1, %l4, %l1 ba,pt %xcc, __handle_preemption_continue - andn %l1, %l4, %l1 + srl %l4, 20, %l4 /* When returning from a NMI (%pil==15) interrupt we want to * avoid running softirqs, doing IRQ tracing, preempting, etc. @@ -67,7 +99,13 @@ rtrap_nmi: ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 andn %l1, %l4, %l1 srl %l4, 20, %l4 ba,pt %xcc, rtrap_no_irq_enable - wrpr %l4, %pil + nop + /* Do not actually set the %pil here. We will do that + * below after we clear PSTATE_IE in the %pstate register. + * If we re-enable interrupts here, we can recurse down + * the hardirq stack potentially endlessly, causing a + * stack overflow. + */ .align 64 .globl rtrap_irq, rtrap, irqsz_patchme, rtrap_xcall @@ -216,10 +254,19 @@ rt_continue: ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 rdpr %otherwin, %l2 srl %l1, 3, %l1 - wrpr %l2, %g0, %canrestore +661: wrpr %l2, %g0, %canrestore + .section .fast_win_ctrl_1insn_patch, "ax" + .word 661b + .word 0x89880000 ! normalw + .previous + wrpr %l1, %g0, %wstate brnz,pt %l2, user_rtt_restore - wrpr %g0, %g0, %otherwin +661: wrpr %g0, %g0, %otherwin + .section .fast_win_ctrl_1insn_patch, "ax" + .word 661b + nop + .previous ldx [%g6 + TI_FLAGS], %g3 wr %g0, ASI_AIUP, %asi @@ -229,53 +276,19 @@ rt_continue: ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 bne,pt %xcc, user_rtt_fill_32bit wrpr %g1, %cwp ba,a,pt %xcc, user_rtt_fill_64bit + nop -user_rtt_fill_fixup: - rdpr %cwp, %g1 - add %g1, 1, %g1 - wrpr %g1, 0x0, %cwp - - rdpr %wstate, %g2 - sll %g2, 3, %g2 - wrpr %g2, 0x0, %wstate - - /* We know %canrestore and %otherwin are both zero. */ - - sethi %hi(sparc64_kern_pri_context), %g2 - ldx [%g2 + %lo(sparc64_kern_pri_context)], %g2 - mov PRIMARY_CONTEXT, %g1 - -661: stxa %g2, [%g1] ASI_DMMU - .section .sun4v_1insn_patch, "ax" - .word 661b - stxa %g2, [%g1] ASI_MMU - .previous - - sethi %hi(KERNBASE), %g1 - flush %g1 - - or %g4, FAULT_CODE_WINFIXUP, %g4 - stb %g4, [%g6 + TI_FAULT_CODE] - stx %g5, [%g6 + TI_FAULT_ADDR] - - mov %g6, %l1 - wrpr %g0, 0x0, %tl - -661: nop - .section .sun4v_1insn_patch, "ax" - .word 661b - SET_GL(0) - .previous +user_rtt_fill_fixup_dax: + ba,pt %xcc, user_rtt_fill_fixup_common + mov 1, %g3 - wrpr %g0, RTRAP_PSTATE, %pstate +user_rtt_fill_fixup_mna: + ba,pt %xcc, user_rtt_fill_fixup_common + mov 2, %g3 - mov %l1, %g6 - ldx [%g6 + TI_TASK], %g4 - LOAD_PER_CPU_BASE(%g5, %g6, %g1, %g2, %g3) - call do_sparc64_fault - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop +user_rtt_fill_fixup: + ba,pt %xcc, user_rtt_fill_fixup_common + clr %g3 user_rtt_pre_restore: add %g1, 1, %g1 @@ -297,7 +310,7 @@ kern_rtt_restore: retry to_kernel: -#ifdef CONFIG_PREEMPT +#ifdef CONFIG_PREEMPTION ldsw [%g6 + TI_PRE_COUNT], %l5 brnz %l5, kern_fpucheck ldx [%g6 + TI_FLAGS], %l5 @@ -306,12 +319,10 @@ to_kernel: nop cmp %l4, 0 bne,pn %xcc, kern_fpucheck - sethi %hi(PREEMPT_ACTIVE), %l6 - stw %l6, [%g6 + TI_PRE_COUNT] - call schedule + nop + call preempt_schedule_irq nop ba,pt %xcc, rtrap - stw %g0, [%g6 + TI_PRE_COUNT] #endif kern_fpucheck: ldub [%g6 + TI_FPDEPTH], %l5 brz,pt %l5, rt_continue diff --git a/arch/sparc/kernel/sbus.c b/arch/sparc/kernel/sbus.c index be5bdf93c767..0bababf6f2bc 100644 --- a/arch/sparc/kernel/sbus.c +++ b/arch/sparc/kernel/sbus.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * sbus.c: UltraSparc SBUS controller support. * @@ -13,7 +14,9 @@ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/numa.h> #include <asm/page.h> #include <asm/io.h> @@ -66,8 +69,8 @@ void sbus_set_sbus64(struct device *dev, int bursts) regs = of_get_property(op->dev.of_node, "reg", NULL); if (!regs) { - printk(KERN_ERR "sbus_set_sbus64: Cannot find regs for %s\n", - op->dev.of_node->full_name); + printk(KERN_ERR "sbus_set_sbus64: Cannot find regs for %pOF\n", + op->dev.of_node); return; } slot = regs->which_io; @@ -560,7 +563,7 @@ static void __init sbus_iommu_init(struct platform_device *op) op->dev.archdata.iommu = iommu; op->dev.archdata.stc = strbuf; - op->dev.archdata.numa_node = -1; + op->dev.archdata.numa_node = NUMA_NO_NODE; reg_base = regs + SYSIO_IOMMUREG_BASE; iommu->iommu_control = reg_base + IOMMU_CONTROL; diff --git a/arch/sparc/kernel/setup.c b/arch/sparc/kernel/setup.c new file mode 100644 index 000000000000..4975867d9001 --- /dev/null +++ b/arch/sparc/kernel/setup.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <asm/setup.h> +#include <linux/sysctl.h> + +static const struct ctl_table sparc_sysctl_table[] = { + { + .procname = "reboot-cmd", + .data = reboot_command, + .maxlen = 256, + .mode = 0644, + .proc_handler = proc_dostring, + }, + { + .procname = "stop-a", + .data = &stop_a_enabled, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "scons-poweroff", + .data = &scons_pwroff, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#ifdef CONFIG_SPARC64 + { + .procname = "tsb-ratio", + .data = &sysctl_tsb_ratio, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif +}; + + +static int __init init_sparc_sysctls(void) +{ + register_sysctl_init("kernel", sparc_sysctl_table); + return 0; +} + +arch_initcall(init_sparc_sysctls); diff --git a/arch/sparc/kernel/setup_32.c b/arch/sparc/kernel/setup_32.c index 1434526970a6..704375c061e7 100644 --- a/arch/sparc/kernel/setup_32.c +++ b/arch/sparc/kernel/setup_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * linux/arch/sparc/kernel/setup.c * @@ -16,7 +17,6 @@ #include <linux/initrd.h> #include <asm/smp.h> #include <linux/user.h> -#include <linux/screen_info.h> #include <linux/delay.h> #include <linux/fs.h> #include <linux/seq_file.h> @@ -33,12 +33,12 @@ #include <linux/kdebug.h> #include <linux/export.h> #include <linux/start_kernel.h> +#include <uapi/linux/mount.h> #include <asm/io.h> #include <asm/processor.h> #include <asm/oplib.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/traps.h> #include <asm/vaddrs.h> #include <asm/mbus.h> @@ -50,26 +50,12 @@ #include "kernel.h" -struct screen_info screen_info = { - 0, 0, /* orig-x, orig-y */ - 0, /* unused */ - 0, /* orig-video-page */ - 0, /* orig-video-mode */ - 128, /* orig-video-cols */ - 0,0,0, /* ega_ax, ega_bx, ega_cx */ - 54, /* orig-video-lines */ - 0, /* orig-video-isVGA */ - 16 /* orig-video-points */ -}; - /* Typing sync at the prom prompt calls the function pointed to by * romvec->pv_synchook which I set to the following function. * This should sync all filesystems and return, for now it just * prints out pretty messages and returns. */ -extern unsigned long trapbase; - /* Pretty sick eh? */ static void prom_sync_me(void) { @@ -81,13 +67,13 @@ static void prom_sync_me(void) __asm__ __volatile__("wr %0, 0x0, %%tbr\n\t" "nop\n\t" "nop\n\t" - "nop\n\t" : : "r" (&trapbase)); + "nop\n\t" : : "r" (&trapbase[0])); prom_printf("PROM SYNC COMMAND...\n"); - show_free_areas(0); + show_mem(); if (!is_idle_task(current)) { local_irq_enable(); - sys_sync(); + ksys_sync(); local_irq_disable(); } prom_printf("Returning to prom\n"); @@ -109,7 +95,7 @@ unsigned long cmdline_memory_size __initdata = 0; unsigned char boot_cpu_id = 0xff; /* 0xff will make it into DATA section... */ static void -prom_console_write(struct console *con, const char *s, unsigned n) +prom_console_write(struct console *con, const char *s, unsigned int n) { prom_write(s, n); } @@ -150,7 +136,7 @@ static void __init boot_flags_init(char *commands) { while (*commands) { /* Move to the start of the next "argument". */ - while (*commands && *commands == ' ') + while (*commands == ' ') commands++; /* Process any command switches, otherwise skip it. */ @@ -267,7 +253,6 @@ static __init void leon_patch(void) } struct tt_entry *sparc_ttable; -struct pt_regs fake_swapper_regs; /* Called from head_32.S - before we have setup anything * in the kernel. Be very careful with what you do here. @@ -300,43 +285,38 @@ void __init setup_arch(char **cmdline_p) int i; unsigned long highest_paddr; - sparc_ttable = (struct tt_entry *) &trapbase; + sparc_ttable = &trapbase[0]; /* Initialize PROM console and command line. */ *cmdline_p = prom_getbootargs(); - strlcpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE); + strscpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE); parse_early_param(); boot_flags_init(*cmdline_p); register_console(&prom_early_console); - printk("ARCH: "); switch(sparc_cpu_model) { case sun4m: - printk("SUN4M\n"); + pr_info("ARCH: SUN4M\n"); break; case sun4d: - printk("SUN4D\n"); + pr_info("ARCH: SUN4D\n"); break; case sun4e: - printk("SUN4E\n"); + pr_info("ARCH: SUN4E\n"); break; case sun4u: - printk("SUN4U\n"); + pr_info("ARCH: SUN4U\n"); break; case sparc_leon: - printk("LEON\n"); + pr_info("ARCH: LEON\n"); break; default: - printk("UNKNOWN!\n"); + pr_info("ARCH: UNKNOWN!\n"); break; } -#ifdef CONFIG_DUMMY_CONSOLE - conswitchp = &dummy_con; -#endif - idprom_init(); load_mmu(); @@ -359,20 +339,16 @@ void __init setup_arch(char **cmdline_p) ROOT_DEV = old_decode_dev(root_dev); #ifdef CONFIG_BLK_DEV_RAM rd_image_start = ram_flags & RAMDISK_IMAGE_START_MASK; - rd_prompt = ((ram_flags & RAMDISK_PROMPT_FLAG) != 0); - rd_doload = ((ram_flags & RAMDISK_LOAD_FLAG) != 0); #endif prom_setsync(prom_sync_me); - if((boot_flags&BOOTME_DEBUG) && (linux_dbvec!=0) && + if((boot_flags & BOOTME_DEBUG) && (linux_dbvec != NULL) && ((*(short *)linux_dbvec) != -1)) { printk("Booted under KADB. Syncing trap table.\n"); (*(linux_dbvec->teach_debugger))(); } - init_task.thread.kregs = &fake_swapper_regs; - /* Run-time patch instructions to match the cpu model */ per_cpu_patch(); @@ -423,3 +399,10 @@ static int __init topology_init(void) } subsys_initcall(topology_init); + +#if defined(CONFIG_SPARC32) && !defined(CONFIG_SMP) +void __init arch_cpu_finalize_init(void) +{ + cpu_data(0).udelay_val = loops_per_jiffy; +} +#endif diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c index 13785547e435..63615f5c99b4 100644 --- a/arch/sparc/kernel/setup_64.c +++ b/arch/sparc/kernel/setup_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * linux/arch/sparc64/kernel/setup.c * @@ -14,7 +15,6 @@ #include <linux/ptrace.h> #include <asm/smp.h> #include <linux/user.h> -#include <linux/screen_info.h> #include <linux/delay.h> #include <linux/fs.h> #include <linux/seq_file.h> @@ -30,12 +30,14 @@ #include <linux/cpu.h> #include <linux/initrd.h> #include <linux/module.h> +#include <linux/start_kernel.h> +#include <linux/memblock.h> +#include <uapi/linux/mount.h> #include <asm/io.h> #include <asm/processor.h> #include <asm/oplib.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/idprom.h> #include <asm/head.h> #include <asm/starfire.h> @@ -49,6 +51,8 @@ #include <asm/elf.h> #include <asm/mdesc.h> #include <asm/cacheflush.h> +#include <asm/dma.h> +#include <asm/irq.h> #ifdef CONFIG_IP_PNP #include <net/ipconfig.h> @@ -63,20 +67,8 @@ DEFINE_SPINLOCK(ns87303_lock); EXPORT_SYMBOL(ns87303_lock); -struct screen_info screen_info = { - 0, 0, /* orig-x, orig-y */ - 0, /* unused */ - 0, /* orig-video-page */ - 0, /* orig-video-mode */ - 128, /* orig-video-cols */ - 0, 0, 0, /* unused, ega_bx, unused */ - 54, /* orig-video-lines */ - 0, /* orig-video-isVGA */ - 16 /* orig-video-points */ -}; - static void -prom_console_write(struct console *con, const char *s, unsigned n) +prom_console_write(struct console *con, const char *s, unsigned int n) { prom_write(s, n); } @@ -91,7 +83,7 @@ static struct console prom_early_console = { .index = -1, }; -/* +/* * Process kernel command line switches that are specific to the * SPARC or that require special low-level processing. */ @@ -129,7 +121,7 @@ static void __init boot_flags_init(char *commands) { while (*commands) { /* Move to the start of the next "argument". */ - while (*commands && *commands == ' ') + while (*commands == ' ') commands++; /* Process any command switches, otherwise skip it. */ @@ -141,21 +133,9 @@ static void __init boot_flags_init(char *commands) process_switch(*commands++); continue; } - if (!strncmp(commands, "mem=", 4)) { - /* - * "mem=XXX[kKmM]" overrides the PROM-reported - * memory size. - */ - cmdline_memory_size = simple_strtoul(commands + 4, - &commands, 0); - if (*commands == 'K' || *commands == 'k') { - cmdline_memory_size <<= 10; - commands++; - } else if (*commands=='M' || *commands=='m') { - cmdline_memory_size <<= 20; - commands++; - } - } + if (!strncmp(commands, "mem=", 4)) + cmdline_memory_size = memparse(commands + 4, &commands); + while (*commands && *commands != ' ') commands++; } @@ -172,9 +152,7 @@ extern int root_mountflags; char reboot_command[COMMAND_LINE_SIZE]; -static struct pt_regs fake_swapper_regs = { { 0, }, 0, 0, 0, 0 }; - -void __init per_cpu_patch(void) +static void __init per_cpu_patch(void) { struct cpuid_patch_entry *p; unsigned long ver; @@ -266,7 +244,25 @@ void sun4v_patch_2insn_range(struct sun4v_2insn_patch_entry *start, } } -void __init sun4v_patch(void) +void sun_m7_patch_2insn_range(struct sun4v_2insn_patch_entry *start, + struct sun4v_2insn_patch_entry *end) +{ + while (start < end) { + unsigned long addr = start->addr; + + *(unsigned int *) (addr + 0) = start->insns[0]; + wmb(); + __asm__ __volatile__("flush %0" : : "r" (addr + 0)); + + *(unsigned int *) (addr + 4) = start->insns[1]; + wmb(); + __asm__ __volatile__("flush %0" : : "r" (addr + 4)); + + start++; + } +} + +static void __init sun4v_patch(void) { extern void sun4v_hvapi_init(void); @@ -279,6 +275,24 @@ void __init sun4v_patch(void) sun4v_patch_2insn_range(&__sun4v_2insn_patch, &__sun4v_2insn_patch_end); + switch (sun4v_chip_type) { + case SUN4V_CHIP_SPARC_M7: + case SUN4V_CHIP_SPARC_M8: + case SUN4V_CHIP_SPARC_SN: + sun4v_patch_1insn_range(&__sun_m7_1insn_patch, + &__sun_m7_1insn_patch_end); + sun_m7_patch_2insn_range(&__sun_m7_2insn_patch, + &__sun_m7_2insn_patch_end); + break; + default: + break; + } + + if (sun4v_chip_type != SUN4V_CHIP_NIAGARA1) { + sun4v_patch_1insn_range(&__fast_win_ctrl_1insn_patch, + &__fast_win_ctrl_1insn_patch_end); + } + sun4v_hvapi_init(); } @@ -335,14 +349,27 @@ static void __init pause_patch(void) } } -#ifdef CONFIG_SMP -void __init boot_cpu_id_too_large(int cpu) +void __init start_early_boot(void) { - prom_printf("Serious problem, boot cpu id (%d) >= NR_CPUS (%d)\n", - cpu, NR_CPUS); - prom_halt(); + int cpu; + + check_if_starfire(); + per_cpu_patch(); + sun4v_patch(); + smp_init_cpu_poke(); + + cpu = hard_smp_processor_id(); + if (cpu >= NR_CPUS) { + prom_printf("Serious problem, boot cpu id (%d) >= NR_CPUS (%d)\n", + cpu, NR_CPUS); + prom_halt(); + } + current_thread_info()->cpu = cpu; + + time_init_early(); + prom_init_report(); + start_kernel(); } -#endif /* On Ultra, we support all of the v8 capabilities. */ unsigned long sparc64_elf_hwcap = (HWCAP_SPARC_FLUSH | HWCAP_SPARC_STBAR | @@ -359,7 +386,8 @@ static const char *hwcaps[] = { */ "mul32", "div32", "fsmuld", "v8plus", "popc", "vis", "vis2", "ASIBlkInit", "fmaf", "vis3", "hpc", "random", "trans", "fjfmau", - "ima", "cspare", "pause", "cbcond", + "ima", "cspare", "pause", "cbcond", NULL /*reserved for crypto */, + "adp", }; static const char *crypto_hwcaps[] = { @@ -375,7 +403,7 @@ void cpucap_info(struct seq_file *m) seq_puts(m, "cpucaps\t\t: "); for (i = 0; i < ARRAY_SIZE(hwcaps); i++) { unsigned long bit = 1UL << i; - if (caps & bit) { + if (hwcaps[i] && (caps & bit)) { seq_printf(m, "%s%s", printed ? "," : "", hwcaps[i]); printed++; @@ -429,7 +457,7 @@ static void __init report_hwcaps(unsigned long caps) for (i = 0; i < ARRAY_SIZE(hwcaps); i++) { unsigned long bit = 1UL << i; - if (caps & bit) + if (hwcaps[i] && (caps & bit)) report_one_hwcap(&printed, hwcaps[i]); } if (caps & HWCAP_SPARC_CRYPTO) @@ -464,7 +492,7 @@ static unsigned long __init mdesc_cpu_hwcap_list(void) for (i = 0; i < ARRAY_SIZE(hwcaps); i++) { unsigned long bit = 1UL << i; - if (!strcmp(prop, hwcaps[i])) { + if (hwcaps[i] && !strcmp(prop, hwcaps[i])) { caps |= bit; break; } @@ -499,12 +527,22 @@ static void __init init_sparc64_elf_hwcap(void) sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || - sun4v_chip_type == SUN4V_CHIP_NIAGARA5) + sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M6 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M7 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M8 || + sun4v_chip_type == SUN4V_CHIP_SPARC_SN || + sun4v_chip_type == SUN4V_CHIP_SPARC64X) cap |= HWCAP_SPARC_BLKINIT; if (sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || - sun4v_chip_type == SUN4V_CHIP_NIAGARA5) + sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M6 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M7 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M8 || + sun4v_chip_type == SUN4V_CHIP_SPARC_SN || + sun4v_chip_type == SUN4V_CHIP_SPARC64X) cap |= HWCAP_SPARC_N2; } @@ -530,13 +568,23 @@ static void __init init_sparc64_elf_hwcap(void) if (sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || - sun4v_chip_type == SUN4V_CHIP_NIAGARA5) + sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M6 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M7 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M8 || + sun4v_chip_type == SUN4V_CHIP_SPARC_SN || + sun4v_chip_type == SUN4V_CHIP_SPARC64X) cap |= (AV_SPARC_VIS | AV_SPARC_VIS2 | AV_SPARC_ASI_BLK_INIT | AV_SPARC_POPC); if (sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || - sun4v_chip_type == SUN4V_CHIP_NIAGARA5) + sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M6 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M7 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M8 || + sun4v_chip_type == SUN4V_CHIP_SPARC_SN || + sun4v_chip_type == SUN4V_CHIP_SPARC64X) cap |= (AV_SPARC_VIS3 | AV_SPARC_HPC | AV_SPARC_FMAF); } @@ -551,11 +599,31 @@ static void __init init_sparc64_elf_hwcap(void) pause_patch(); } +static void __init alloc_irqstack_bootmem(void) +{ + unsigned int i, node; + + for_each_possible_cpu(i) { + node = cpu_to_node(i); + + softirq_stack[i] = memblock_alloc_node(THREAD_SIZE, + THREAD_SIZE, node); + if (!softirq_stack[i]) + panic("%s: Failed to allocate %lu bytes align=%lx nid=%d\n", + __func__, THREAD_SIZE, THREAD_SIZE, node); + hardirq_stack[i] = memblock_alloc_node(THREAD_SIZE, + THREAD_SIZE, node); + if (!hardirq_stack[i]) + panic("%s: Failed to allocate %lu bytes align=%lx nid=%d\n", + __func__, THREAD_SIZE, THREAD_SIZE, node); + } +} + void __init setup_arch(char **cmdline_p) { /* Initialize PROM console and command line. */ *cmdline_p = prom_getbootargs(); - strlcpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE); + strscpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE); parse_early_param(); boot_flags_init(*cmdline_p); @@ -565,13 +633,9 @@ void __init setup_arch(char **cmdline_p) register_console(&prom_early_console); if (tlb_type == hypervisor) - printk("ARCH: SUN4V\n"); + pr_info("ARCH: SUN4V\n"); else - printk("ARCH: SUN4U\n"); - -#ifdef CONFIG_DUMMY_CONSOLE - conswitchp = &dummy_con; -#endif + pr_info("ARCH: SUN4U\n"); idprom_init(); @@ -580,17 +644,13 @@ void __init setup_arch(char **cmdline_p) ROOT_DEV = old_decode_dev(root_dev); #ifdef CONFIG_BLK_DEV_RAM rd_image_start = ram_flags & RAMDISK_IMAGE_START_MASK; - rd_prompt = ((ram_flags & RAMDISK_PROMPT_FLAG) != 0); - rd_doload = ((ram_flags & RAMDISK_LOAD_FLAG) != 0); #endif - task_thread_info(&init_task)->kregs = &fake_swapper_regs; - #ifdef CONFIG_IP_PNP if (!ic_set_manually) { phandle chosen = prom_finddevice("/chosen"); u32 cl, sv, gw; - + cl = prom_getintdefault (chosen, "client-ip", 0); sv = prom_getintdefault (chosen, "server-ip", 0); gw = prom_getintdefault (chosen, "gateway-ip", 0); @@ -611,6 +671,12 @@ void __init setup_arch(char **cmdline_p) paging_init(); init_sparc64_elf_hwcap(); + /* + * Once the OF device tree and MDESC have been setup and nr_cpus has + * been parsed, we know the list of possible cpus. Therefore we can + * allocate the IRQ stacks. + */ + alloc_irqstack_bootmem(); } extern int stop_a_enabled; diff --git a/arch/sparc/kernel/signal32.c b/arch/sparc/kernel/signal32.c index b524f91dd0e5..a23cdd7459bb 100644 --- a/arch/sparc/kernel/signal32.c +++ b/arch/sparc/kernel/signal32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* arch/sparc64/kernel/signal32.c * * Copyright (C) 1991, 1992 Linus Torvalds @@ -19,11 +20,9 @@ #include <linux/binfmts.h> #include <linux/compat.h> #include <linux/bitops.h> -#include <linux/tracehook.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/ptrace.h> -#include <asm/pgtable.h> #include <asm/psrcompat.h> #include <asm/fpumacro.h> #include <asm/visasm.h> @@ -31,6 +30,7 @@ #include <asm/switch_to.h> #include "sigutil.h" +#include "kernel.h" /* This magic should be in g_upper[0] for all upper parts * to be valid. @@ -68,73 +68,16 @@ struct rt_signal_frame32 { /* __siginfo_rwin_t * */u32 rwin_save; } __attribute__((aligned(8))); -int copy_siginfo_to_user32(compat_siginfo_t __user *to, siginfo_t *from) -{ - int err; - - if (!access_ok(VERIFY_WRITE, to, sizeof(compat_siginfo_t))) - return -EFAULT; - - /* If you change siginfo_t structure, please be sure - this code is fixed accordingly. - It should never copy any pad contained in the structure - to avoid security leaks, but must copy the generic - 3 ints plus the relevant union member. - This routine must convert siginfo from 64bit to 32bit as well - at the same time. */ - err = __put_user(from->si_signo, &to->si_signo); - err |= __put_user(from->si_errno, &to->si_errno); - err |= __put_user((short)from->si_code, &to->si_code); - if (from->si_code < 0) - err |= __copy_to_user(&to->_sifields._pad, &from->_sifields._pad, SI_PAD_SIZE); - else { - switch (from->si_code >> 16) { - case __SI_TIMER >> 16: - err |= __put_user(from->si_tid, &to->si_tid); - err |= __put_user(from->si_overrun, &to->si_overrun); - err |= __put_user(from->si_int, &to->si_int); - break; - case __SI_CHLD >> 16: - err |= __put_user(from->si_utime, &to->si_utime); - err |= __put_user(from->si_stime, &to->si_stime); - err |= __put_user(from->si_status, &to->si_status); - default: - err |= __put_user(from->si_pid, &to->si_pid); - err |= __put_user(from->si_uid, &to->si_uid); - break; - case __SI_FAULT >> 16: - err |= __put_user(from->si_trapno, &to->si_trapno); - err |= __put_user((unsigned long)from->si_addr, &to->si_addr); - break; - case __SI_POLL >> 16: - err |= __put_user(from->si_band, &to->si_band); - err |= __put_user(from->si_fd, &to->si_fd); - break; - case __SI_RT >> 16: /* This is not generated by the kernel as of now. */ - case __SI_MESGQ >> 16: - err |= __put_user(from->si_pid, &to->si_pid); - err |= __put_user(from->si_uid, &to->si_uid); - err |= __put_user(from->si_int, &to->si_int); - break; - } - } - return err; -} - -/* CAUTION: This is just a very minimalist implementation for the - * sake of compat_sys_rt_sigqueueinfo() +/* Checks if the fp is valid. We always build signal frames which are + * 16-byte aligned, therefore we can always enforce that the restore + * frame has that property as well. */ -int copy_siginfo_from_user32(siginfo_t *to, compat_siginfo_t __user *from) +static bool invalid_frame_pointer(void __user *fp, int fplen) { - if (!access_ok(VERIFY_WRITE, from, sizeof(compat_siginfo_t))) - return -EFAULT; - - if (copy_from_user(to, from, 3*sizeof(int)) || - copy_from_user(to->_sifields._pad, from->_sifields._pad, - SI_PAD_SIZE)) - return -EFAULT; - - return 0; + if ((((unsigned long) fp) & 15) || + ((unsigned long)fp) > 0x100000000ULL - fplen) + return true; + return false; } void do_sigreturn32(struct pt_regs *regs) @@ -142,14 +85,14 @@ void do_sigreturn32(struct pt_regs *regs) struct signal_frame32 __user *sf; compat_uptr_t fpu_save; compat_uptr_t rwin_save; - unsigned int psr; - unsigned pc, npc; + unsigned int psr, ufp; + unsigned int pc, npc; sigset_t set; - unsigned seta[_COMPAT_NSIG_WORDS]; + compat_sigset_t seta; int err, i; /* Always make any pending restarted system calls return -EINTR */ - current_thread_info()->restart_block.fn = do_no_restart_syscall; + current->restart_block.fn = do_no_restart_syscall; synchronize_user_stack(); @@ -157,11 +100,16 @@ void do_sigreturn32(struct pt_regs *regs) sf = (struct signal_frame32 __user *) regs->u_regs[UREG_FP]; /* 1. Make sure we are not getting garbage from the user */ - if (!access_ok(VERIFY_READ, sf, sizeof(*sf)) || - (((unsigned long) sf) & 3)) + if (invalid_frame_pointer(sf, sizeof(*sf))) goto segv; - if (get_user(pc, &sf->info.si_regs.pc) || + if (get_user(ufp, &sf->info.si_regs.u_regs[UREG_FP])) + goto segv; + + if (ufp & 0x7) + goto segv; + + if (__get_user(pc, &sf->info.si_regs.pc) || __get_user(npc, &sf->info.si_regs.npc)) goto segv; @@ -209,47 +157,47 @@ void do_sigreturn32(struct pt_regs *regs) if (restore_rwin_state(compat_ptr(rwin_save))) goto segv; } - err |= __get_user(seta[0], &sf->info.si_mask); - err |= copy_from_user(seta+1, &sf->extramask, + err |= __get_user(seta.sig[0], &sf->info.si_mask); + err |= copy_from_user(&seta.sig[1], &sf->extramask, (_COMPAT_NSIG_WORDS - 1) * sizeof(unsigned int)); if (err) goto segv; - switch (_NSIG_WORDS) { - case 4: set.sig[3] = seta[6] + (((long)seta[7]) << 32); - case 3: set.sig[2] = seta[4] + (((long)seta[5]) << 32); - case 2: set.sig[1] = seta[2] + (((long)seta[3]) << 32); - case 1: set.sig[0] = seta[0] + (((long)seta[1]) << 32); - } + + set.sig[0] = seta.sig[0] + (((long)seta.sig[1]) << 32); set_current_blocked(&set); return; segv: - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); } asmlinkage void do_rt_sigreturn32(struct pt_regs *regs) { struct rt_signal_frame32 __user *sf; - unsigned int psr, pc, npc; + unsigned int psr, pc, npc, ufp; compat_uptr_t fpu_save; compat_uptr_t rwin_save; sigset_t set; - compat_sigset_t seta; int err, i; /* Always make any pending restarted system calls return -EINTR */ - current_thread_info()->restart_block.fn = do_no_restart_syscall; + current->restart_block.fn = do_no_restart_syscall; synchronize_user_stack(); regs->u_regs[UREG_FP] &= 0x00000000ffffffffUL; sf = (struct rt_signal_frame32 __user *) regs->u_regs[UREG_FP]; /* 1. Make sure we are not getting garbage from the user */ - if (!access_ok(VERIFY_READ, sf, sizeof(*sf)) || - (((unsigned long) sf) & 3)) + if (invalid_frame_pointer(sf, sizeof(*sf))) goto segv; - if (get_user(pc, &sf->regs.pc) || + if (get_user(ufp, &sf->regs.u_regs[UREG_FP])) + goto segv; + + if (ufp & 0x7) + goto segv; + + if (__get_user(pc, &sf->regs.pc) || __get_user(npc, &sf->regs.npc)) goto segv; @@ -292,7 +240,7 @@ asmlinkage void do_rt_sigreturn32(struct pt_regs *regs) err |= __get_user(fpu_save, &sf->fpu_save); if (!err && fpu_save) err |= restore_fpu_state(regs, compat_ptr(fpu_save)); - err |= copy_from_user(&seta, &sf->mask, sizeof(compat_sigset_t)); + err |= get_compat_sigset(&set, &sf->mask); err |= compat_restore_altstack(&sf->stack); if (err) goto segv; @@ -303,24 +251,10 @@ asmlinkage void do_rt_sigreturn32(struct pt_regs *regs) goto segv; } - switch (_NSIG_WORDS) { - case 4: set.sig[3] = seta.sig[6] + (((long)seta.sig[7]) << 32); - case 3: set.sig[2] = seta.sig[4] + (((long)seta.sig[5]) << 32); - case 2: set.sig[1] = seta.sig[2] + (((long)seta.sig[3]) << 32); - case 1: set.sig[0] = seta.sig[0] + (((long)seta.sig[1]) << 32); - } set_current_blocked(&set); return; segv: - force_sig(SIGSEGV, current); -} - -/* Checks if the fp is valid */ -static int invalid_frame_pointer(void __user *fp, int fplen) -{ - if ((((unsigned long) fp) & 7) || ((unsigned long)fp) > 0x100000000ULL - fplen) - return 1; - return 0; + force_sig(SIGSEGV); } static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) @@ -363,6 +297,7 @@ static void flush_signal_insns(unsigned long address) unsigned long pstate, paddr; pte_t *ptep, pte; pgd_t *pgdp; + p4d_t *p4dp; pud_t *pudp; pmd_t *pmdp; @@ -382,7 +317,10 @@ static void flush_signal_insns(unsigned long address) pgdp = pgd_offset(current->mm, address); if (pgd_none(*pgdp)) goto out_irqs_on; - pudp = pud_offset(pgdp, address); + p4dp = p4d_offset(pgdp, address); + if (p4d_none(*p4dp)) + goto out_irqs_on; + pudp = pud_offset(p4dp, address); if (pud_none(*pudp)) goto out_irqs_on; pmdp = pmd_offset(pudp, address); @@ -390,6 +328,8 @@ static void flush_signal_insns(unsigned long address) goto out_irqs_on; ptep = pte_offset_map(pmdp, address); + if (!ptep) + goto out_irqs_on; pte = *ptep; if (!pte_present(pte)) goto out_unmap; @@ -417,7 +357,7 @@ static int setup_frame32(struct ksignal *ksig, struct pt_regs *regs, void __user *tail; int sigframe_size; u32 psr; - unsigned int seta[_COMPAT_NSIG_WORDS]; + compat_sigset_t seta; /* 1. Make sure everything is clean */ synchronize_user_stack(); @@ -435,7 +375,11 @@ static int setup_frame32(struct ksignal *ksig, struct pt_regs *regs, get_sigframe(ksig, regs, sigframe_size); if (invalid_frame_pointer(sf, sigframe_size)) { - do_exit(SIGILL); + if (show_unhandled_signals) + pr_info("%s[%d] bad frame in setup_frame32: %08lx TPC %08lx O7 %08lx\n", + current->comm, current->pid, (unsigned long)sf, + regs->tpc, regs->u_regs[UREG_I7]); + force_sigsegv(ksig->sig); return -EINVAL; } @@ -481,24 +425,20 @@ static int setup_frame32(struct ksignal *ksig, struct pt_regs *regs, err |= __put_user(0, &sf->rwin_save); } - switch (_NSIG_WORDS) { - case 4: seta[7] = (oldset->sig[3] >> 32); - seta[6] = oldset->sig[3]; - case 3: seta[5] = (oldset->sig[2] >> 32); - seta[4] = oldset->sig[2]; - case 2: seta[3] = (oldset->sig[1] >> 32); - seta[2] = oldset->sig[1]; - case 1: seta[1] = (oldset->sig[0] >> 32); - seta[0] = oldset->sig[0]; - } - err |= __put_user(seta[0], &sf->info.si_mask); - err |= __copy_to_user(sf->extramask, seta + 1, + /* If these change we need to know - assignments to seta relies on these sizes */ + BUILD_BUG_ON(_NSIG_WORDS != 1); + BUILD_BUG_ON(_COMPAT_NSIG_WORDS != 2); + seta.sig[1] = (oldset->sig[0] >> 32); + seta.sig[0] = oldset->sig[0]; + + err |= __put_user(seta.sig[0], &sf->info.si_mask); + err |= __copy_to_user(sf->extramask, &seta.sig[1], (_COMPAT_NSIG_WORDS - 1) * sizeof(unsigned int)); if (!wsaved) { - err |= copy_in_user((u32 __user *)sf, - (u32 __user *)(regs->u_regs[UREG_FP]), - sizeof(struct reg_window32)); + err |= raw_copy_in_user((u32 __user *)sf, + (u32 __user *)(regs->u_regs[UREG_FP]), + sizeof(struct reg_window32)); } else { struct reg_window *rp; @@ -552,7 +492,6 @@ static int setup_rt_frame32(struct ksignal *ksig, struct pt_regs *regs, void __user *tail; int sigframe_size; u32 psr; - compat_sigset_t seta; /* 1. Make sure everything is clean */ synchronize_user_stack(); @@ -570,7 +509,11 @@ static int setup_rt_frame32(struct ksignal *ksig, struct pt_regs *regs, get_sigframe(ksig, regs, sigframe_size); if (invalid_frame_pointer(sf, sigframe_size)) { - do_exit(SIGILL); + if (show_unhandled_signals) + pr_info("%s[%d] bad frame in setup_rt_frame32: %08lx TPC %08lx O7 %08lx\n", + current->comm, current->pid, (unsigned long)sf, + regs->tpc, regs->u_regs[UREG_I7]); + force_sigsegv(ksig->sig); return -EINVAL; } @@ -622,22 +565,12 @@ static int setup_rt_frame32(struct ksignal *ksig, struct pt_regs *regs, /* Setup sigaltstack */ err |= __compat_save_altstack(&sf->stack, regs->u_regs[UREG_FP]); - switch (_NSIG_WORDS) { - case 4: seta.sig[7] = (oldset->sig[3] >> 32); - seta.sig[6] = oldset->sig[3]; - case 3: seta.sig[5] = (oldset->sig[2] >> 32); - seta.sig[4] = oldset->sig[2]; - case 2: seta.sig[3] = (oldset->sig[1] >> 32); - seta.sig[2] = oldset->sig[1]; - case 1: seta.sig[1] = (oldset->sig[0] >> 32); - seta.sig[0] = oldset->sig[0]; - } - err |= __copy_to_user(&sf->mask, &seta, sizeof(compat_sigset_t)); + err |= put_compat_sigset(&sf->mask, oldset, sizeof(compat_sigset_t)); if (!wsaved) { - err |= copy_in_user((u32 __user *)sf, - (u32 __user *)(regs->u_regs[UREG_FP]), - sizeof(struct reg_window32)); + err |= raw_copy_in_user((u32 __user *)sf, + (u32 __user *)(regs->u_regs[UREG_FP]), + sizeof(struct reg_window32)); } else { struct reg_window *rp; @@ -714,7 +647,7 @@ static inline void syscall_restart32(unsigned long orig_i0, struct pt_regs *regs case ERESTARTSYS: if (!(sa->sa_flags & SA_RESTART)) goto no_system_call_restart; - /* fallthrough */ + fallthrough; case ERESTARTNOINTR: regs->u_regs[UREG_I0] = orig_i0; regs->tpc -= 4; @@ -754,6 +687,7 @@ void do_signal32(struct pt_regs * regs) regs->tpc -= 4; regs->tnpc -= 4; pt_regs_clear_syscall(regs); + fallthrough; case ERESTART_RESTARTBLOCK: regs->u_regs[UREG_G1] = __NR_restart_syscall; regs->tpc -= 4; @@ -812,3 +746,41 @@ asmlinkage int do_sys32_sigstack(u32 u_ssptr, u32 u_ossptr, unsigned long sp) out: return ret; } + +/* + * Compile-time assertions for siginfo_t offsets. Check NSIG* as well, as + * changes likely come with new fields that should be added below. + */ +static_assert(NSIGILL == 11); +static_assert(NSIGFPE == 15); +static_assert(NSIGSEGV == 10); +static_assert(NSIGBUS == 5); +static_assert(NSIGTRAP == 6); +static_assert(NSIGCHLD == 6); +static_assert(NSIGSYS == 2); +static_assert(sizeof(compat_siginfo_t) == 128); +static_assert(__alignof__(compat_siginfo_t) == 4); +static_assert(offsetof(compat_siginfo_t, si_signo) == 0x00); +static_assert(offsetof(compat_siginfo_t, si_errno) == 0x04); +static_assert(offsetof(compat_siginfo_t, si_code) == 0x08); +static_assert(offsetof(compat_siginfo_t, si_pid) == 0x0c); +static_assert(offsetof(compat_siginfo_t, si_uid) == 0x10); +static_assert(offsetof(compat_siginfo_t, si_tid) == 0x0c); +static_assert(offsetof(compat_siginfo_t, si_overrun) == 0x10); +static_assert(offsetof(compat_siginfo_t, si_status) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_utime) == 0x18); +static_assert(offsetof(compat_siginfo_t, si_stime) == 0x1c); +static_assert(offsetof(compat_siginfo_t, si_value) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_int) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_ptr) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_addr) == 0x0c); +static_assert(offsetof(compat_siginfo_t, si_trapno) == 0x10); +static_assert(offsetof(compat_siginfo_t, si_addr_lsb) == 0x10); +static_assert(offsetof(compat_siginfo_t, si_lower) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_upper) == 0x18); +static_assert(offsetof(compat_siginfo_t, si_pkey) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_perf_data) == 0x10); +static_assert(offsetof(compat_siginfo_t, si_perf_type) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_perf_flags) == 0x18); +static_assert(offsetof(compat_siginfo_t, si_band) == 0x0c); +static_assert(offsetof(compat_siginfo_t, si_fd) == 0x10); diff --git a/arch/sparc/kernel/signal_32.c b/arch/sparc/kernel/signal_32.c index 7d5d8e1f8415..478014d2e59b 100644 --- a/arch/sparc/kernel/signal_32.c +++ b/arch/sparc/kernel/signal_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* linux/arch/sparc/kernel/signal.c * * Copyright (C) 1991, 1992 Linus Torvalds @@ -18,16 +19,15 @@ #include <linux/smp.h> #include <linux/binfmts.h> /* do_coredum */ #include <linux/bitops.h> -#include <linux/tracehook.h> +#include <linux/resume_user_mode.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/ptrace.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> #include <asm/cacheflush.h> /* flush_sig_insns */ #include <asm/switch_to.h> #include "sigutil.h" +#include "kernel.h" extern void fpsave(unsigned long *fpregs, unsigned long *fsr, void *fpqueue, unsigned long *fpqdepth); @@ -59,27 +59,42 @@ struct rt_signal_frame { #define SF_ALIGNEDSZ (((sizeof(struct signal_frame) + 7) & (~7))) #define RT_ALIGNEDSZ (((sizeof(struct rt_signal_frame) + 7) & (~7))) +/* Checks if the fp is valid. We always build signal frames which are + * 16-byte aligned, therefore we can always enforce that the restore + * frame has that property as well. + */ +static inline bool invalid_frame_pointer(void __user *fp, int fplen) +{ + if ((((unsigned long) fp) & 15) || !access_ok(fp, fplen)) + return true; + + return false; +} + asmlinkage void do_sigreturn(struct pt_regs *regs) { + unsigned long up_psr, pc, npc, ufp; struct signal_frame __user *sf; - unsigned long up_psr, pc, npc; sigset_t set; __siginfo_fpu_t __user *fpu_save; __siginfo_rwin_t __user *rwin_save; int err; /* Always make any pending restarted system calls return -EINTR */ - current_thread_info()->restart_block.fn = do_no_restart_syscall; + current->restart_block.fn = do_no_restart_syscall; synchronize_user_stack(); sf = (struct signal_frame __user *) regs->u_regs[UREG_FP]; /* 1. Make sure we are not getting garbage from the user */ - if (!access_ok(VERIFY_READ, sf, sizeof(*sf))) + if (invalid_frame_pointer(sf, sizeof(*sf))) goto segv_and_exit; - if (((unsigned long) sf) & 3) + if (get_user(ufp, &sf->info.si_regs.u_regs[UREG_FP])) + goto segv_and_exit; + + if (ufp & 0x7) goto segv_and_exit; err = __get_user(pc, &sf->info.si_regs.pc); @@ -120,13 +135,13 @@ asmlinkage void do_sigreturn(struct pt_regs *regs) return; segv_and_exit: - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); } asmlinkage void do_rt_sigreturn(struct pt_regs *regs) { struct rt_signal_frame __user *sf; - unsigned int psr, pc, npc; + unsigned int psr, pc, npc, ufp; __siginfo_fpu_t __user *fpu_save; __siginfo_rwin_t __user *rwin_save; sigset_t set; @@ -134,8 +149,13 @@ asmlinkage void do_rt_sigreturn(struct pt_regs *regs) synchronize_user_stack(); sf = (struct rt_signal_frame __user *) regs->u_regs[UREG_FP]; - if (!access_ok(VERIFY_READ, sf, sizeof(*sf)) || - (((unsigned long) sf) & 0x03)) + if (invalid_frame_pointer(sf, sizeof(*sf))) + goto segv; + + if (get_user(ufp, &sf->regs.u_regs[UREG_FP])) + goto segv; + + if (ufp & 0x7) goto segv; err = __get_user(pc, &sf->regs.pc); @@ -174,16 +194,7 @@ asmlinkage void do_rt_sigreturn(struct pt_regs *regs) set_current_blocked(&set); return; segv: - force_sig(SIGSEGV, current); -} - -/* Checks if the fp is valid */ -static inline int invalid_frame_pointer(void __user *fp, int fplen) -{ - if ((((unsigned long) fp) & 7) || !__access_ok((unsigned long)fp, fplen)) - return 1; - - return 0; + force_sig(SIGSEGV); } static inline void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) @@ -233,7 +244,7 @@ static int setup_frame(struct ksignal *ksig, struct pt_regs *regs, get_sigframe(ksig, regs, sigframe_size); if (invalid_frame_pointer(sf, sigframe_size)) { - do_exit(SIGILL); + force_exit_sig(SIGILL); return -EINVAL; } @@ -325,7 +336,7 @@ static int setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs, sf = (struct rt_signal_frame __user *) get_sigframe(ksig, regs, sigframe_size); if (invalid_frame_pointer(sf, sigframe_size)) { - do_exit(SIGILL); + force_exit_sig(SIGILL); return -EINVAL; } @@ -341,7 +352,7 @@ static int setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs, err |= __put_user(0, &sf->extra_size); if (psr & PSR_EF) { - __siginfo_fpu_t *fp = tail; + __siginfo_fpu_t __user *fp = tail; tail += sizeof(*fp); err |= save_fpu_state(regs, fp); err |= __put_user(fp, &sf->fpu_save); @@ -349,7 +360,7 @@ static int setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs, err |= __put_user(0, &sf->fpu_save); } if (wsaved) { - __siginfo_rwin_t *rwp = tail; + __siginfo_rwin_t __user *rwp = tail; tail += sizeof(*rwp); err |= save_rwin_state(wsaved, rwp); err |= __put_user(rwp, &sf->rwin_save); @@ -389,8 +400,8 @@ static int setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs, else { regs->u_regs[UREG_I7] = (unsigned long)(&(sf->insns[0]) - 2); - /* mov __NR_sigreturn, %g1 */ - err |= __put_user(0x821020d8, &sf->insns[0]); + /* mov __NR_rt_sigreturn, %g1 */ + err |= __put_user(0x82102065, &sf->insns[0]); /* t 0x10 */ err |= __put_user(0x91d02010, &sf->insns[1]); @@ -429,7 +440,7 @@ static inline void syscall_restart(unsigned long orig_i0, struct pt_regs *regs, case ERESTARTSYS: if (!(sa->sa_flags & SA_RESTART)) goto no_system_call_restart; - /* fallthrough */ + fallthrough; case ERESTARTNOINTR: regs->u_regs[UREG_I0] = orig_i0; regs->pc -= 4; @@ -462,7 +473,7 @@ static void do_signal(struct pt_regs *regs, unsigned long orig_i0) * * %g7 is used as the "thread register". %g6 is not used in * any fixed manner. %g6 is used as a scratch register and - * a compiler temporary, but it's value is never used across + * a compiler temporary, but its value is never used across * a system call. Therefore %g6 is usable for orig_i0 storage. */ if (pt_regs_is_syscall(regs) && (regs->psr & PSR_C)) @@ -495,6 +506,7 @@ static void do_signal(struct pt_regs *regs, unsigned long orig_i0) regs->pc -= 4; regs->npc -= 4; pt_regs_clear_syscall(regs); + fallthrough; case ERESTART_RESTARTBLOCK: regs->u_regs[UREG_G1] = __NR_restart_syscall; regs->pc -= 4; @@ -509,17 +521,15 @@ static void do_signal(struct pt_regs *regs, unsigned long orig_i0) void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long thread_info_flags) { - if (thread_info_flags & _TIF_SIGPENDING) + if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) do_signal(regs, orig_i0); - if (thread_info_flags & _TIF_NOTIFY_RESUME) { - clear_thread_flag(TIF_NOTIFY_RESUME); - tracehook_notify_resume(regs); - } + if (thread_info_flags & _TIF_NOTIFY_RESUME) + resume_user_mode_work(regs); } -asmlinkage int -do_sys_sigstack(struct sigstack __user *ssptr, struct sigstack __user *ossptr, - unsigned long sp) +asmlinkage int do_sys_sigstack(struct sigstack __user *ssptr, + struct sigstack __user *ossptr, + unsigned long sp) { int ret = -EFAULT; diff --git a/arch/sparc/kernel/signal_64.c b/arch/sparc/kernel/signal_64.c index 35923e8abd82..2d64566a1f88 100644 --- a/arch/sparc/kernel/signal_64.c +++ b/arch/sparc/kernel/signal_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * arch/sparc64/kernel/signal.c * @@ -8,25 +9,22 @@ * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ -#ifdef CONFIG_COMPAT -#include <linux/compat.h> /* for compat_old_sigset_t */ -#endif #include <linux/sched.h> #include <linux/kernel.h> #include <linux/signal.h> #include <linux/errno.h> #include <linux/wait.h> #include <linux/ptrace.h> -#include <linux/tracehook.h> +#include <linux/resume_user_mode.h> #include <linux/unistd.h> #include <linux/mm.h> #include <linux/tty.h> #include <linux/binfmts.h> #include <linux/bitops.h> +#include <linux/context_tracking.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/ptrace.h> -#include <asm/pgtable.h> #include <asm/fpumacro.h> #include <asm/uctx.h> #include <asm/siginfo.h> @@ -34,22 +32,24 @@ #include <asm/switch_to.h> #include <asm/cacheflush.h> -#include "entry.h" -#include "systbls.h" #include "sigutil.h" +#include "systbls.h" +#include "kernel.h" +#include "entry.h" /* {set, get}context() needed for 64-bit SparcLinux userland. */ asmlinkage void sparc64_set_context(struct pt_regs *regs) { struct ucontext __user *ucp = (struct ucontext __user *) regs->u_regs[UREG_I0]; + enum ctx_state prev_state = exception_enter(); mc_gregset_t __user *grp; unsigned long pc, npc, tstate; unsigned long fp, i7; unsigned char fenab; int err; - flush_user_windows(); + synchronize_user_stack(); if (get_thread_wsaved() || (((unsigned long)ucp) & (sizeof(unsigned long)-1)) || (!__access_ok(ucp, sizeof(*ucp)))) @@ -129,16 +129,19 @@ asmlinkage void sparc64_set_context(struct pt_regs *regs) } if (err) goto do_sigsegv; - +out: + exception_exit(prev_state); return; do_sigsegv: - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); + goto out; } asmlinkage void sparc64_get_context(struct pt_regs *regs) { struct ucontext __user *ucp = (struct ucontext __user *) regs->u_regs[UREG_I0]; + enum ctx_state prev_state = exception_enter(); mc_gregset_t __user *grp; mcontext_t __user *mcp; unsigned long fp, i7; @@ -220,10 +223,23 @@ asmlinkage void sparc64_get_context(struct pt_regs *regs) } if (err) goto do_sigsegv; - +out: + exception_exit(prev_state); return; do_sigsegv: - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); + goto out; +} + +/* Checks if the fp is valid. We always build rt signal frames which + * are 16-byte aligned, therefore we can always enforce that the + * restore frame has that property as well. + */ +static bool invalid_frame_pointer(void __user *fp) +{ + if (((unsigned long) fp) & 15) + return true; + return false; } struct rt_signal_frame { @@ -238,25 +254,31 @@ struct rt_signal_frame { void do_rt_sigreturn(struct pt_regs *regs) { + unsigned long tpc, tnpc, tstate, ufp; struct rt_signal_frame __user *sf; - unsigned long tpc, tnpc, tstate; __siginfo_fpu_t __user *fpu_save; __siginfo_rwin_t __user *rwin_save; sigset_t set; int err; /* Always make any pending restarted system calls return -EINTR */ - current_thread_info()->restart_block.fn = do_no_restart_syscall; + current->restart_block.fn = do_no_restart_syscall; synchronize_user_stack (); sf = (struct rt_signal_frame __user *) (regs->u_regs [UREG_FP] + STACK_BIAS); /* 1. Make sure we are not getting garbage from the user */ - if (((unsigned long) sf) & 3) + if (invalid_frame_pointer(sf)) + goto segv; + + if (get_user(ufp, &sf->regs.u_regs[UREG_FP])) goto segv; - err = get_user(tpc, &sf->regs.tpc); + if ((ufp + STACK_BIAS) & 0x7) + goto segv; + + err = __get_user(tpc, &sf->regs.tpc); err |= __get_user(tnpc, &sf->regs.tnpc); if (test_thread_flag(TIF_32BIT)) { tpc &= 0xffffffff; @@ -297,15 +319,7 @@ void do_rt_sigreturn(struct pt_regs *regs) set_current_blocked(&set); return; segv: - force_sig(SIGSEGV, current); -} - -/* Checks if the fp is valid */ -static int invalid_frame_pointer(void __user *fp) -{ - if (((unsigned long) fp) & 15) - return 1; - return 0; + force_sig(SIGSEGV); } static inline void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) @@ -355,7 +369,11 @@ setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs) get_sigframe(ksig, regs, sf_size); if (invalid_frame_pointer (sf)) { - do_exit(SIGILL); /* won't return, actually */ + if (show_unhandled_signals) + pr_info("%s[%d] bad frame in setup_rt_frame: %016lx TPC %016lx O7 %016lx\n", + current->comm, current->pid, (unsigned long)sf, + regs->tpc, regs->u_regs[UREG_I7]); + force_sigsegv(ksig->sig); return -EINVAL; } @@ -388,10 +406,10 @@ setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs) err |= copy_to_user(&sf->mask, sigmask_to_save(), sizeof(sigset_t)); if (!wsaved) { - err |= copy_in_user((u64 __user *)sf, - (u64 __user *)(regs->u_regs[UREG_FP] + - STACK_BIAS), - sizeof(struct reg_window)); + err |= raw_copy_in_user((u64 __user *)sf, + (u64 __user *)(regs->u_regs[UREG_FP] + + STACK_BIAS), + sizeof(struct reg_window)); } else { struct reg_window *rp; @@ -443,7 +461,7 @@ static inline void syscall_restart(unsigned long orig_i0, struct pt_regs *regs, case ERESTARTSYS: if (!(sa->sa_flags & SA_RESTART)) goto no_system_call_restart; - /* fallthrough */ + fallthrough; case ERESTARTNOINTR: regs->u_regs[UREG_I0] = orig_i0; regs->tpc -= 4; @@ -476,7 +494,7 @@ static void do_signal(struct pt_regs *regs, unsigned long orig_i0) * * %g7 is used as the "thread register". %g6 is not used in * any fixed manner. %g6 is used as a scratch register and - * a compiler temporary, but it's value is never used across + * a compiler temporary, but its value is never used across * a system call. Therefore %g6 is usable for orig_i0 storage. */ if (pt_regs_is_syscall(regs) && @@ -485,7 +503,6 @@ static void do_signal(struct pt_regs *regs, unsigned long orig_i0) #ifdef CONFIG_COMPAT if (test_thread_flag(TIF_32BIT)) { - extern void do_signal32(struct pt_regs *); do_signal32(regs); return; } @@ -515,6 +532,7 @@ static void do_signal(struct pt_regs *regs, unsigned long orig_i0) regs->tpc -= 4; regs->tnpc -= 4; pt_regs_clear_syscall(regs); + fallthrough; case ERESTART_RESTARTBLOCK: regs->u_regs[UREG_G1] = __NR_restart_syscall; regs->tpc -= 4; @@ -528,11 +546,50 @@ static void do_signal(struct pt_regs *regs, unsigned long orig_i0) void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long thread_info_flags) { - if (thread_info_flags & _TIF_SIGPENDING) + user_exit(); + if (thread_info_flags & _TIF_UPROBE) + uprobe_notify_resume(regs); + if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) do_signal(regs, orig_i0); - if (thread_info_flags & _TIF_NOTIFY_RESUME) { - clear_thread_flag(TIF_NOTIFY_RESUME); - tracehook_notify_resume(regs); - } + if (thread_info_flags & _TIF_NOTIFY_RESUME) + resume_user_mode_work(regs); + user_enter(); } +/* + * Compile-time assertions for siginfo_t offsets. Check NSIG* as well, as + * changes likely come with new fields that should be added below. + */ +static_assert(NSIGILL == 11); +static_assert(NSIGFPE == 15); +static_assert(NSIGSEGV == 10); +static_assert(NSIGBUS == 5); +static_assert(NSIGTRAP == 6); +static_assert(NSIGCHLD == 6); +static_assert(NSIGSYS == 2); +static_assert(sizeof(siginfo_t) == 128); +static_assert(__alignof__(siginfo_t) == 8); +static_assert(offsetof(siginfo_t, si_signo) == 0x00); +static_assert(offsetof(siginfo_t, si_errno) == 0x04); +static_assert(offsetof(siginfo_t, si_code) == 0x08); +static_assert(offsetof(siginfo_t, si_pid) == 0x10); +static_assert(offsetof(siginfo_t, si_uid) == 0x14); +static_assert(offsetof(siginfo_t, si_tid) == 0x10); +static_assert(offsetof(siginfo_t, si_overrun) == 0x14); +static_assert(offsetof(siginfo_t, si_status) == 0x18); +static_assert(offsetof(siginfo_t, si_utime) == 0x20); +static_assert(offsetof(siginfo_t, si_stime) == 0x28); +static_assert(offsetof(siginfo_t, si_value) == 0x18); +static_assert(offsetof(siginfo_t, si_int) == 0x18); +static_assert(offsetof(siginfo_t, si_ptr) == 0x18); +static_assert(offsetof(siginfo_t, si_addr) == 0x10); +static_assert(offsetof(siginfo_t, si_trapno) == 0x18); +static_assert(offsetof(siginfo_t, si_addr_lsb) == 0x18); +static_assert(offsetof(siginfo_t, si_lower) == 0x20); +static_assert(offsetof(siginfo_t, si_upper) == 0x28); +static_assert(offsetof(siginfo_t, si_pkey) == 0x20); +static_assert(offsetof(siginfo_t, si_perf_data) == 0x18); +static_assert(offsetof(siginfo_t, si_perf_type) == 0x20); +static_assert(offsetof(siginfo_t, si_perf_flags) == 0x24); +static_assert(offsetof(siginfo_t, si_band) == 0x10); +static_assert(offsetof(siginfo_t, si_fd) == 0x14); diff --git a/arch/sparc/kernel/sigutil.h b/arch/sparc/kernel/sigutil.h index d223aa432bb6..21d332d8bddd 100644 --- a/arch/sparc/kernel/sigutil.h +++ b/arch/sparc/kernel/sigutil.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SIGUTIL_H #define _SIGUTIL_H diff --git a/arch/sparc/kernel/sigutil_32.c b/arch/sparc/kernel/sigutil_32.c index 0f6eebe71e6c..f25c6daa9f52 100644 --- a/arch/sparc/kernel/sigutil_32.c +++ b/arch/sparc/kernel/sigutil_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/kernel.h> #include <linux/types.h> #include <linux/thread_info.h> @@ -48,6 +49,10 @@ int save_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) int restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) { int err; + + if (((unsigned long) fpu) & 3) + return -EFAULT; + #ifdef CONFIG_SMP if (test_tsk_thread_flag(current, TIF_USEDFPU)) regs->psr &= ~PSR_EF; @@ -60,7 +65,7 @@ int restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) set_used_math(); clear_tsk_thread_flag(current, TIF_USEDFPU); - if (!access_ok(VERIFY_READ, fpu, sizeof(*fpu))) + if (!access_ok(fpu, sizeof(*fpu))) return -EFAULT; err = __copy_from_user(¤t->thread.float_regs[0], &fpu->si_float_regs[0], @@ -97,7 +102,10 @@ int restore_rwin_state(__siginfo_rwin_t __user *rp) struct thread_info *t = current_thread_info(); int i, wsaved, err; - __get_user(wsaved, &rp->wsaved); + if (((unsigned long) rp) & 3) + return -EFAULT; + + get_user(wsaved, &rp->wsaved); if (wsaved > NSWINS) return -EFAULT; diff --git a/arch/sparc/kernel/sigutil_64.c b/arch/sparc/kernel/sigutil_64.c index 387834a9c56a..512e4639e4a1 100644 --- a/arch/sparc/kernel/sigutil_64.c +++ b/arch/sparc/kernel/sigutil_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/kernel.h> #include <linux/types.h> #include <linux/thread_info.h> @@ -37,7 +38,10 @@ int restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) unsigned long fprs; int err; - err = __get_user(fprs, &fpu->si_fprs); + if (((unsigned long) fpu) & 7) + return -EFAULT; + + err = get_user(fprs, &fpu->si_fprs); fprs_write(0); regs->tstate &= ~TSTATE_PEF; if (fprs & FPRS_DL) @@ -72,7 +76,10 @@ int restore_rwin_state(__siginfo_rwin_t __user *rp) struct thread_info *t = current_thread_info(); int i, wsaved, err; - __get_user(wsaved, &rp->wsaved); + if (((unsigned long) rp) & 7) + return -EFAULT; + + get_user(wsaved, &rp->wsaved); if (wsaved > NSWINS) return -EFAULT; diff --git a/arch/sparc/kernel/smp_32.c b/arch/sparc/kernel/smp_32.c index e3f2b81c23f1..87eaa7719fa2 100644 --- a/arch/sparc/kernel/smp_32.c +++ b/arch/sparc/kernel/smp_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* smp.c: Sparc SMP support. * * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) @@ -20,6 +21,7 @@ #include <linux/seq_file.h> #include <linux/cache.h> #include <linux/delay.h> +#include <linux/profile.h> #include <linux/cpu.h> #include <asm/ptrace.h> @@ -27,8 +29,6 @@ #include <asm/irq.h> #include <asm/page.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> #include <asm/oplib.h> #include <asm/cacheflush.h> #include <asm/tlbflush.h> @@ -39,7 +39,7 @@ #include "kernel.h" #include "irq.h" -volatile unsigned long cpu_callin_map[NR_CPUS] __cpuinitdata = {0,}; +volatile unsigned long cpu_callin_map[NR_CPUS] = {0,}; cpumask_t smp_commenced_mask = CPU_MASK_NONE; @@ -53,7 +53,7 @@ const struct sparc32_ipi_ops *sparc32_ipi_ops; * instruction which is much better... */ -void __cpuinit smp_store_cpu_info(int id) +void smp_store_cpu_info(int id) { int cpu_node; int mid; @@ -67,7 +67,7 @@ void __cpuinit smp_store_cpu_info(int id) mid = cpu_get_hwmid(cpu_node); if (mid < 0) { - printk(KERN_NOTICE "No MID found for CPU%d at node 0x%08d", id, cpu_node); + printk(KERN_NOTICE "No MID found for CPU%d at node 0x%08x", id, cpu_node); mid = 0; } cpu_data(id).mid = mid; @@ -75,8 +75,6 @@ void __cpuinit smp_store_cpu_info(int id) void __init smp_cpus_done(unsigned int max_cpus) { - extern void smp4m_smp_done(void); - extern void smp4d_smp_done(void); unsigned long bogosum = 0; int cpu, num = 0; @@ -120,9 +118,9 @@ void cpu_panic(void) panic("SMP bolixed\n"); } -struct linux_prom_registers smp_penguin_ctable __cpuinitdata = { 0 }; +struct linux_prom_registers smp_penguin_ctable = { 0 }; -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { /* * CPU model dependent way of implementing IPI generation targeting @@ -176,15 +174,8 @@ void smp_call_function_interrupt(void) irq_exit(); } -int setup_profiling_timer(unsigned int multiplier) -{ - return -EINVAL; -} - void __init smp_prepare_cpus(unsigned int max_cpus) { - extern void __init smp4m_boot_cpus(void); - extern void __init smp4d_boot_cpus(void); int i, cpuid, extra; printk("Entering SMP Mode...\n"); @@ -259,10 +250,8 @@ void __init smp_prepare_boot_cpu(void) set_cpu_possible(cpuid, true); } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { - extern int __cpuinit smp4m_boot_one_cpu(int, struct task_struct *); - extern int __cpuinit smp4d_boot_one_cpu(int, struct task_struct *); int ret=0; switch(sparc_cpu_model) { @@ -297,7 +286,7 @@ int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) return ret; } -void __cpuinit arch_cpu_pre_starting(void *arg) +static void arch_cpu_pre_starting(void *arg) { local_ops->cache_all(); local_ops->tlb_all(); @@ -317,7 +306,7 @@ void __cpuinit arch_cpu_pre_starting(void *arg) } } -void __cpuinit arch_cpu_pre_online(void *arg) +static void arch_cpu_pre_online(void *arg) { unsigned int cpuid = hard_smp_processor_id(); @@ -344,7 +333,7 @@ void __cpuinit arch_cpu_pre_online(void *arg) } } -void __cpuinit sparc_start_secondary(void *arg) +static void sparc_start_secondary(void *arg) { unsigned int cpu; @@ -354,12 +343,9 @@ void __cpuinit sparc_start_secondary(void *arg) */ arch_cpu_pre_starting(arg); - preempt_disable(); cpu = smp_processor_id(); - /* Invoke the CPU_STARTING notifier callbacks */ notify_cpu_starting(cpu); - arch_cpu_pre_online(arg); /* Set the CPU in the cpu_online_mask */ @@ -369,13 +355,13 @@ void __cpuinit sparc_start_secondary(void *arg) local_irq_enable(); wmb(); - cpu_startup_entry(CPUHP_ONLINE); + cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); /* We should never reach here! */ BUG(); } -void __cpuinit smp_callin(void) +void smp_callin(void) { sparc_start_secondary(NULL); } diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index 77539eda928c..5cbd6ed5ef6f 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* smp.c: Sparc64 SMP support. * * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net) @@ -5,7 +6,8 @@ #include <linux/export.h> #include <linux/kernel.h> -#include <linux/sched.h> +#include <linux/sched/mm.h> +#include <linux/sched/hotplug.h> #include <linux/mm.h> #include <linux/pagemap.h> #include <linux/threads.h> @@ -20,11 +22,12 @@ #include <linux/cache.h> #include <linux/jiffies.h> #include <linux/profile.h> -#include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/vmalloc.h> #include <linux/ftrace.h> #include <linux/cpu.h> #include <linux/slab.h> +#include <linux/kgdb.h> #include <asm/head.h> #include <asm/ptrace.h> @@ -35,15 +38,16 @@ #include <asm/hvtramp.h> #include <asm/io.h> #include <asm/timer.h> +#include <asm/setup.h> #include <asm/irq.h> #include <asm/irq_regs.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/oplib.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/starfire.h> #include <asm/tlb.h> +#include <asm/pgalloc.h> #include <asm/sections.h> #include <asm/prom.h> #include <asm/mdesc.h> @@ -52,18 +56,28 @@ #include <asm/pcr.h> #include "cpumap.h" - -int sparc64_multi_core __read_mostly; +#include "kernel.h" DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE; cpumask_t cpu_core_map[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = CPU_MASK_NONE }; +cpumask_t cpu_core_sib_map[NR_CPUS] __read_mostly = { + [0 ... NR_CPUS-1] = CPU_MASK_NONE }; + +cpumask_t cpu_core_sib_cache_map[NR_CPUS] __read_mostly = { + [0 ... NR_CPUS - 1] = CPU_MASK_NONE }; + EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); EXPORT_SYMBOL(cpu_core_map); +EXPORT_SYMBOL(cpu_core_sib_map); +EXPORT_SYMBOL(cpu_core_sib_cache_map); static cpumask_t smp_commenced_mask; +static DEFINE_PER_CPU(bool, poke); +static bool cpu_poke; + void smp_info(struct seq_file *m) { int i; @@ -87,7 +101,7 @@ extern void setup_sparc64_timer(void); static volatile unsigned long callin_flag = 0; -void __cpuinit smp_callin(void) +void smp_callin(void) { int cpuid = hard_smp_processor_id(); @@ -113,7 +127,7 @@ void __cpuinit smp_callin(void) current_thread_info()->new_child = 0; /* Attach to the address space of init_task. */ - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; /* inform the notifiers about the new cpu */ @@ -123,12 +137,10 @@ void __cpuinit smp_callin(void) rmb(); set_cpu_online(cpuid, true); - local_irq_enable(); - /* idle thread is expected to have preempt disabled */ - preempt_disable(); + local_irq_enable(); - cpu_startup_entry(CPUHP_ONLINE); + cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); } void cpu_panic(void) @@ -150,7 +162,7 @@ void cpu_panic(void) #define NUM_ROUNDS 64 /* magic value */ #define NUM_ITERS 5 /* likewise */ -static DEFINE_SPINLOCK(itc_sync_lock); +static DEFINE_RAW_SPINLOCK(itc_sync_lock); static unsigned long go[SLAVE + 1]; #define DEBUG_TICK_SYNC 0 @@ -258,7 +270,7 @@ static void smp_synchronize_one_tick(int cpu) go[MASTER] = 0; membar_safe("#StoreLoad"); - spin_lock_irqsave(&itc_sync_lock, flags); + raw_spin_lock_irqsave(&itc_sync_lock, flags); { for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) { while (!go[MASTER]) @@ -269,19 +281,12 @@ static void smp_synchronize_one_tick(int cpu) membar_safe("#StoreLoad"); } } - spin_unlock_irqrestore(&itc_sync_lock, flags); + raw_spin_unlock_irqrestore(&itc_sync_lock, flags); } #if defined(CONFIG_SUN_LDOMS) && defined(CONFIG_HOTPLUG_CPU) -/* XXX Put this in some common place. XXX */ -static unsigned long kimage_addr_to_ra(void *p) -{ - unsigned long val = (unsigned long) p; - - return kern_base + (val - KERNBASE); -} - -static void __cpuinit ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg, void **descrp) +static void ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg, + void **descrp) { extern unsigned long sparc64_ttable_tl0; extern unsigned long kern_locked_tte_data; @@ -292,9 +297,7 @@ static void __cpuinit ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread unsigned long hv_err; int i; - hdesc = kzalloc(sizeof(*hdesc) + - (sizeof(struct hvtramp_mapping) * - num_kernel_image_mappings - 1), + hdesc = kzalloc(struct_size(hdesc, maps, num_kernel_image_mappings), GFP_KERNEL); if (!hdesc) { printk(KERN_ERR "ldom_startcpu_cpuid: Cannot allocate " @@ -342,7 +345,7 @@ extern unsigned long sparc64_cpu_startup; */ static struct thread_info *cpu_new_thread = NULL; -static int __cpuinit smp_boot_one_cpu(unsigned int cpu, struct task_struct *idle) +static int smp_boot_one_cpu(unsigned int cpu, struct task_struct *idle) { unsigned long entry = (unsigned long)(&sparc64_cpu_startup); @@ -618,22 +621,48 @@ retry: } } -/* Multi-cpu list version. */ +#define CPU_MONDO_COUNTER(cpuid) (cpu_mondo_counter[cpuid]) +#define MONDO_USEC_WAIT_MIN 2 +#define MONDO_USEC_WAIT_MAX 100 +#define MONDO_RETRY_LIMIT 500000 + +/* Multi-cpu list version. + * + * Deliver xcalls to 'cnt' number of cpus in 'cpu_list'. + * Sometimes not all cpus receive the mondo, requiring us to re-send + * the mondo until all cpus have received, or cpus are truly stuck + * unable to receive mondo, and we timeout. + * Occasionally a target cpu strand is borrowed briefly by hypervisor to + * perform guest service, such as PCIe error handling. Consider the + * service time, 1 second overall wait is reasonable for 1 cpu. + * Here two in-between mondo check wait time are defined: 2 usec for + * single cpu quick turn around and up to 100usec for large cpu count. + * Deliver mondo to large number of cpus could take longer, we adjusts + * the retry count as long as target cpus are making forward progress. + */ static void hypervisor_xcall_deliver(struct trap_per_cpu *tb, int cnt) { - int retries, this_cpu, prev_sent, i, saw_cpu_error; + int this_cpu, tot_cpus, prev_sent, i, rem; + int usec_wait, retries, tot_retries; + u16 first_cpu = 0xffff; + unsigned long xc_rcvd = 0; unsigned long status; + int ecpuerror_id = 0; + int enocpu_id = 0; u16 *cpu_list; + u16 cpu; this_cpu = smp_processor_id(); - cpu_list = __va(tb->cpu_list_pa); - - saw_cpu_error = 0; - retries = 0; + usec_wait = cnt * MONDO_USEC_WAIT_MIN; + if (usec_wait > MONDO_USEC_WAIT_MAX) + usec_wait = MONDO_USEC_WAIT_MAX; + retries = tot_retries = 0; + tot_cpus = cnt; prev_sent = 0; + do { - int forward_progress, n_sent; + int n_sent, mondo_delivered, target_cpu_busy; status = sun4v_cpu_mondo_send(cnt, tb->cpu_list_pa, @@ -641,94 +670,113 @@ static void hypervisor_xcall_deliver(struct trap_per_cpu *tb, int cnt) /* HV_EOK means all cpus received the xcall, we're done. */ if (likely(status == HV_EOK)) - break; + goto xcall_done; + + /* If not these non-fatal errors, panic */ + if (unlikely((status != HV_EWOULDBLOCK) && + (status != HV_ECPUERROR) && + (status != HV_ENOCPU))) + goto fatal_errors; /* First, see if we made any forward progress. * + * Go through the cpu_list, count the target cpus that have + * received our mondo (n_sent), and those that did not (rem). + * Re-pack cpu_list with the cpus remain to be retried in the + * front - this simplifies tracking the truly stalled cpus. + * * The hypervisor indicates successful sends by setting * cpu list entries to the value 0xffff. + * + * EWOULDBLOCK means some target cpus did not receive the + * mondo and retry usually helps. + * + * ECPUERROR means at least one target cpu is in error state, + * it's usually safe to skip the faulty cpu and retry. + * + * ENOCPU means one of the target cpu doesn't belong to the + * domain, perhaps offlined which is unexpected, but not + * fatal and it's okay to skip the offlined cpu. */ + rem = 0; n_sent = 0; for (i = 0; i < cnt; i++) { - if (likely(cpu_list[i] == 0xffff)) + cpu = cpu_list[i]; + if (likely(cpu == 0xffff)) { n_sent++; + } else if ((status == HV_ECPUERROR) && + (sun4v_cpu_state(cpu) == HV_CPU_STATE_ERROR)) { + ecpuerror_id = cpu + 1; + } else if (status == HV_ENOCPU && !cpu_online(cpu)) { + enocpu_id = cpu + 1; + } else { + cpu_list[rem++] = cpu; + } } - forward_progress = 0; - if (n_sent > prev_sent) - forward_progress = 1; + /* No cpu remained, we're done. */ + if (rem == 0) + break; - prev_sent = n_sent; + /* Otherwise, update the cpu count for retry. */ + cnt = rem; - /* If we get a HV_ECPUERROR, then one or more of the cpus - * in the list are in error state. Use the cpu_state() - * hypervisor call to find out which cpus are in error state. + /* Record the overall number of mondos received by the + * first of the remaining cpus. */ - if (unlikely(status == HV_ECPUERROR)) { - for (i = 0; i < cnt; i++) { - long err; - u16 cpu; + if (first_cpu != cpu_list[0]) { + first_cpu = cpu_list[0]; + xc_rcvd = CPU_MONDO_COUNTER(first_cpu); + } - cpu = cpu_list[i]; - if (cpu == 0xffff) - continue; + /* Was any mondo delivered successfully? */ + mondo_delivered = (n_sent > prev_sent); + prev_sent = n_sent; - err = sun4v_cpu_state(cpu); - if (err == HV_CPU_STATE_ERROR) { - saw_cpu_error = (cpu + 1); - cpu_list[i] = 0xffff; - } - } - } else if (unlikely(status != HV_EWOULDBLOCK)) - goto fatal_mondo_error; + /* or, was any target cpu busy processing other mondos? */ + target_cpu_busy = (xc_rcvd < CPU_MONDO_COUNTER(first_cpu)); + xc_rcvd = CPU_MONDO_COUNTER(first_cpu); - /* Don't bother rewriting the CPU list, just leave the - * 0xffff and non-0xffff entries in there and the - * hypervisor will do the right thing. - * - * Only advance timeout state if we didn't make any - * forward progress. + /* Retry count is for no progress. If we're making progress, + * reset the retry count. */ - if (unlikely(!forward_progress)) { - if (unlikely(++retries > 10000)) - goto fatal_mondo_timeout; - - /* Delay a little bit to let other cpus catch up - * on their cpu mondo queue work. - */ - udelay(2 * cnt); + if (likely(mondo_delivered || target_cpu_busy)) { + tot_retries += retries; + retries = 0; + } else if (unlikely(retries > MONDO_RETRY_LIMIT)) { + goto fatal_mondo_timeout; } - } while (1); - if (unlikely(saw_cpu_error)) - goto fatal_mondo_cpu_error; + /* Delay a little bit to let other cpus catch up on + * their cpu mondo queue work. + */ + if (!mondo_delivered) + udelay(usec_wait); - return; + retries++; + } while (1); -fatal_mondo_cpu_error: - printk(KERN_CRIT "CPU[%d]: SUN4V mondo cpu error, some target cpus " - "(including %d) were in error state\n", - this_cpu, saw_cpu_error - 1); +xcall_done: + if (unlikely(ecpuerror_id > 0)) { + pr_crit("CPU[%d]: SUN4V mondo cpu error, target cpu(%d) was in error state\n", + this_cpu, ecpuerror_id - 1); + } else if (unlikely(enocpu_id > 0)) { + pr_crit("CPU[%d]: SUN4V mondo cpu error, target cpu(%d) does not belong to the domain\n", + this_cpu, enocpu_id - 1); + } return; +fatal_errors: + /* fatal errors include bad alignment, etc */ + pr_crit("CPU[%d]: Args were cnt(%d) cpulist_pa(%lx) mondo_block_pa(%lx)\n", + this_cpu, tot_cpus, tb->cpu_list_pa, tb->cpu_mondo_block_pa); + panic("Unexpected SUN4V mondo error %lu\n", status); + fatal_mondo_timeout: - printk(KERN_CRIT "CPU[%d]: SUN4V mondo timeout, no forward " - " progress after %d retries.\n", - this_cpu, retries); - goto dump_cpu_list_and_out; - -fatal_mondo_error: - printk(KERN_CRIT "CPU[%d]: Unexpected SUN4V mondo error %lu\n", - this_cpu, status); - printk(KERN_CRIT "CPU[%d]: Args were cnt(%d) cpulist_pa(%lx) " - "mondo_block_pa(%lx)\n", - this_cpu, cnt, tb->cpu_list_pa, tb->cpu_mondo_block_pa); - -dump_cpu_list_and_out: - printk(KERN_CRIT "CPU[%d]: CPU list [ ", this_cpu); - for (i = 0; i < cnt; i++) - printk("%u ", cpu_list[i]); - printk("]\n"); + /* some cpus being non-responsive to the cpu mondo */ + pr_crit("CPU[%d]: SUN4V mondo timeout, cpu(%d) made no forward progress after %d retries. Total target cpus(%d).\n", + this_cpu, first_cpu, (tot_retries + retries), tot_cpus); + panic("SUN4V mondo timeout panic\n"); } static void (*xcall_deliver_impl)(struct trap_per_cpu *, int); @@ -821,13 +869,17 @@ void arch_send_call_function_single_ipi(int cpu) void __irq_entry smp_call_function_client(int irq, struct pt_regs *regs) { clear_softint(1 << irq); + irq_enter(); generic_smp_call_function_interrupt(); + irq_exit(); } void __irq_entry smp_call_function_single_client(int irq, struct pt_regs *regs) { clear_softint(1 << irq); + irq_enter(); generic_smp_call_function_single_interrupt(); + irq_exit(); } static void tsb_sync(void *info) @@ -867,25 +919,26 @@ extern unsigned long xcall_flush_dcache_page_cheetah; #endif extern unsigned long xcall_flush_dcache_page_spitfire; -#ifdef CONFIG_DEBUG_DCFLUSH -extern atomic_t dcpage_flushes; -extern atomic_t dcpage_flushes_xcall; -#endif - -static inline void __local_flush_dcache_page(struct page *page) +static inline void __local_flush_dcache_folio(struct folio *folio) { + unsigned int i, nr = folio_nr_pages(folio); + #ifdef DCACHE_ALIASING_POSSIBLE - __flush_dcache_page(page_address(page), + for (i = 0; i < nr; i++) + __flush_dcache_page(folio_address(folio) + i * PAGE_SIZE, ((tlb_type == spitfire) && - page_mapping(page) != NULL)); + folio_flush_mapping(folio) != NULL)); #else - if (page_mapping(page) != NULL && - tlb_type == spitfire) - __flush_icache_page(__pa(page_address(page))); + if (folio_flush_mapping(folio) != NULL && + tlb_type == spitfire) { + unsigned long pfn = folio_pfn(folio) + for (i = 0; i < nr; i++) + __flush_icache_page((pfn + i) * PAGE_SIZE); + } #endif } -void smp_flush_dcache_page_impl(struct page *page, int cpu) +void smp_flush_dcache_folio_impl(struct folio *folio, int cpu) { int this_cpu; @@ -899,14 +952,14 @@ void smp_flush_dcache_page_impl(struct page *page, int cpu) this_cpu = get_cpu(); if (cpu == this_cpu) { - __local_flush_dcache_page(page); + __local_flush_dcache_folio(folio); } else if (cpu_online(cpu)) { - void *pg_addr = page_address(page); + void *pg_addr = folio_address(folio); u64 data0 = 0; if (tlb_type == spitfire) { data0 = ((u64)&xcall_flush_dcache_page_spitfire); - if (page_mapping(page) != NULL) + if (folio_flush_mapping(folio) != NULL) data0 |= ((u64)1 << 32); } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { #ifdef DCACHE_ALIASING_POSSIBLE @@ -914,18 +967,23 @@ void smp_flush_dcache_page_impl(struct page *page, int cpu) #endif } if (data0) { - xcall_deliver(data0, __pa(pg_addr), - (u64) pg_addr, cpumask_of(cpu)); + unsigned int i, nr = folio_nr_pages(folio); + + for (i = 0; i < nr; i++) { + xcall_deliver(data0, __pa(pg_addr), + (u64) pg_addr, cpumask_of(cpu)); #ifdef CONFIG_DEBUG_DCFLUSH - atomic_inc(&dcpage_flushes_xcall); + atomic_inc(&dcpage_flushes_xcall); #endif + pg_addr += PAGE_SIZE; + } } } put_cpu(); } -void flush_dcache_page_all(struct mm_struct *mm, struct page *page) +void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio) { void *pg_addr; u64 data0; @@ -939,10 +997,10 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page) atomic_inc(&dcpage_flushes); #endif data0 = 0; - pg_addr = page_address(page); + pg_addr = folio_address(folio); if (tlb_type == spitfire) { data0 = ((u64)&xcall_flush_dcache_page_spitfire); - if (page_mapping(page) != NULL) + if (folio_flush_mapping(folio) != NULL) data0 |= ((u64)1 << 32); } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { #ifdef DCACHE_ALIASING_POSSIBLE @@ -950,50 +1008,24 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page) #endif } if (data0) { - xcall_deliver(data0, __pa(pg_addr), - (u64) pg_addr, cpu_online_mask); + unsigned int i, nr = folio_nr_pages(folio); + + for (i = 0; i < nr; i++) { + xcall_deliver(data0, __pa(pg_addr), + (u64) pg_addr, cpu_online_mask); #ifdef CONFIG_DEBUG_DCFLUSH - atomic_inc(&dcpage_flushes_xcall); + atomic_inc(&dcpage_flushes_xcall); #endif + pg_addr += PAGE_SIZE; + } } - __local_flush_dcache_page(page); + __local_flush_dcache_folio(folio); preempt_enable(); } -void __irq_entry smp_new_mmu_context_version_client(int irq, struct pt_regs *regs) -{ - struct mm_struct *mm; - unsigned long flags; - - clear_softint(1 << irq); - - /* See if we need to allocate a new TLB context because - * the version of the one we are using is now out of date. - */ - mm = current->active_mm; - if (unlikely(!mm || (mm == &init_mm))) - return; - - spin_lock_irqsave(&mm->context.lock, flags); - - if (unlikely(!CTX_VALID(mm->context))) - get_new_mmu_context(mm); - - spin_unlock_irqrestore(&mm->context.lock, flags); - - load_secondary_context(mm); - __flush_tlb_mm(CTX_HWBITS(mm->context), - SECONDARY_CONTEXT); -} - -void smp_new_mmu_context_version(void) -{ - smp_cross_call(&xcall_new_mmu_context_version, 0, 0, 0); -} - #ifdef CONFIG_KGDB -void kgdb_roundup_cpus(unsigned long flags) +void kgdb_roundup_cpus(void) { smp_cross_call(&xcall_kgdb_capture, 0, 0, 0); } @@ -1018,38 +1050,9 @@ void smp_fetch_global_pmu(void) * are flush_tlb_*() routines, and these run after flush_cache_*() * which performs the flushw. * - * The SMP TLB coherency scheme we use works as follows: - * - * 1) mm->cpu_vm_mask is a bit mask of which cpus an address - * space has (potentially) executed on, this is the heuristic - * we use to avoid doing cross calls. - * - * Also, for flushing from kswapd and also for clones, we - * use cpu_vm_mask as the list of cpus to make run the TLB. - * - * 2) TLB context numbers are shared globally across all processors - * in the system, this allows us to play several games to avoid - * cross calls. - * - * One invariant is that when a cpu switches to a process, and - * that processes tsk->active_mm->cpu_vm_mask does not have the - * current cpu's bit set, that tlb context is flushed locally. - * - * If the address space is non-shared (ie. mm->count == 1) we avoid - * cross calls when we want to flush the currently running process's - * tlb state. This is done by clearing all cpu bits except the current - * processor's in current->mm->cpu_vm_mask and performing the - * flush locally only. This will force any subsequent cpus which run - * this task to flush the context from the local tlb if the process - * migrates to another cpu (again). - * - * 3) For shared address spaces (threads) and swapping we bite the - * bullet for most cases and perform the cross call (but only to - * the cpus listed in cpu_vm_mask). - * - * The performance gain from "optimizing" away the cross call for threads is - * questionable (in theory the big win for threads is the massive sharing of - * address space state across processors). + * mm->cpu_vm_mask is a bit mask of which cpus an address + * space has (potentially) executed on, this is the heuristic + * we use to limit cross calls. */ /* This currently is only used by the hugetlb arch pre-fault @@ -1059,18 +1062,13 @@ void smp_fetch_global_pmu(void) void smp_flush_tlb_mm(struct mm_struct *mm) { u32 ctx = CTX_HWBITS(mm->context); - int cpu = get_cpu(); - if (atomic_read(&mm->mm_users) == 1) { - cpumask_copy(mm_cpumask(mm), cpumask_of(cpu)); - goto local_flush_and_out; - } + get_cpu(); smp_cross_call_masked(&xcall_flush_tlb_mm, ctx, 0, 0, mm_cpumask(mm)); -local_flush_and_out: __flush_tlb_mm(ctx, SECONDARY_CONTEXT); put_cpu(); @@ -1093,17 +1091,15 @@ void smp_flush_tlb_pending(struct mm_struct *mm, unsigned long nr, unsigned long { u32 ctx = CTX_HWBITS(mm->context); struct tlb_pending_info info; - int cpu = get_cpu(); + + get_cpu(); info.ctx = ctx; info.nr = nr; info.vaddrs = vaddrs; - if (mm == current->mm && atomic_read(&mm->mm_users) == 1) - cpumask_copy(mm_cpumask(mm), cpumask_of(cpu)); - else - smp_call_function_many(mm_cpumask(mm), tlb_pending_func, - &info, 1); + smp_call_function_many(mm_cpumask(mm), tlb_pending_func, + &info, 1); __flush_tlb_pending(ctx, nr, vaddrs); @@ -1113,14 +1109,13 @@ void smp_flush_tlb_pending(struct mm_struct *mm, unsigned long nr, unsigned long void smp_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr) { unsigned long context = CTX_HWBITS(mm->context); - int cpu = get_cpu(); - if (mm == current->mm && atomic_read(&mm->mm_users) == 1) - cpumask_copy(mm_cpumask(mm), cpumask_of(cpu)); - else - smp_cross_call_masked(&xcall_flush_tlb_page, - context, vaddr, 0, - mm_cpumask(mm)); + get_cpu(); + + smp_cross_call_masked(&xcall_flush_tlb_page, + context, vaddr, 0, + mm_cpumask(mm)); + __flush_tlb_page(context, vaddr); put_cpu(); @@ -1148,7 +1143,7 @@ static unsigned long penguins_are_doing_time; void smp_capture(void) { - int result = atomic_add_ret(1, &smp_capture_depth); + int result = atomic_add_return(1, &smp_capture_depth); if (result == 1) { int ncpus = num_online_cpus(); @@ -1205,20 +1200,10 @@ void __irq_entry smp_penguin_jailcell(int irq, struct pt_regs *regs) preempt_enable(); } -/* /proc/profile writes can call this, don't __init it please. */ -int setup_profiling_timer(unsigned int multiplier) -{ - return -EINVAL; -} - void __init smp_prepare_cpus(unsigned int max_cpus) { } -void smp_prepare_boot_cpu(void) -{ -} - void __init smp_setup_processor_id(void) { if (tlb_type == spitfire) @@ -1249,6 +1234,19 @@ void smp_fill_in_sib_core_maps(void) } } + for_each_present_cpu(i) { + unsigned int j; + + for_each_present_cpu(j) { + if (cpu_data(i).max_cache_id == + cpu_data(j).max_cache_id) + cpumask_set_cpu(j, &cpu_core_sib_cache_map[i]); + + if (cpu_data(i).sock_id == cpu_data(j).sock_id) + cpumask_set_cpu(j, &cpu_core_sib_map[i]); + } + } + for_each_present_cpu(i) { unsigned int j; @@ -1266,7 +1264,7 @@ void smp_fill_in_sib_core_maps(void) } } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { int ret = smp_boot_one_cpu(cpu, tidle); @@ -1393,13 +1391,88 @@ void __cpu_die(unsigned int cpu) void __init smp_cpus_done(unsigned int max_cpus) { - pcr_arch_init(); } -void smp_send_reschedule(int cpu) +static void send_cpu_ipi(int cpu) { - xcall_deliver((u64) &xcall_receive_signal, 0, 0, - cpumask_of(cpu)); + xcall_deliver((u64) &xcall_receive_signal, + 0, 0, cpumask_of(cpu)); +} + +void scheduler_poke(void) +{ + if (!cpu_poke) + return; + + if (!__this_cpu_read(poke)) + return; + + __this_cpu_write(poke, false); + set_softint(1 << PIL_SMP_RECEIVE_SIGNAL); +} + +static unsigned long send_cpu_poke(int cpu) +{ + unsigned long hv_err; + + per_cpu(poke, cpu) = true; + hv_err = sun4v_cpu_poke(cpu); + if (hv_err != HV_EOK) { + per_cpu(poke, cpu) = false; + pr_err_ratelimited("%s: sun4v_cpu_poke() fails err=%lu\n", + __func__, hv_err); + } + + return hv_err; +} + +void arch_smp_send_reschedule(int cpu) +{ + if (cpu == smp_processor_id()) { + WARN_ON_ONCE(preemptible()); + set_softint(1 << PIL_SMP_RECEIVE_SIGNAL); + return; + } + + /* Use cpu poke to resume idle cpu if supported. */ + if (cpu_poke && idle_cpu(cpu)) { + unsigned long ret; + + ret = send_cpu_poke(cpu); + if (ret == HV_EOK) + return; + } + + /* Use IPI in following cases: + * - cpu poke not supported + * - cpu not idle + * - send_cpu_poke() returns with error + */ + send_cpu_ipi(cpu); +} + +void smp_init_cpu_poke(void) +{ + unsigned long major; + unsigned long minor; + int ret; + + if (tlb_type != hypervisor) + return; + + ret = sun4v_hvapi_get(HV_GRP_CORE, &major, &minor); + if (ret) { + pr_debug("HV_GRP_CORE is not registered\n"); + return; + } + + if (major == 1 && minor >= 6) { + /* CPU POKE is registered. */ + cpu_poke = true; + return; + } + + pr_debug("CPU_POKE not supported\n"); } void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs) @@ -1408,55 +1481,39 @@ void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs) scheduler_ipi(); } -/* This is a nop because we capture all other cpus - * anyways when making the PROM active. - */ -void smp_send_stop(void) +static void stop_this_cpu(void *dummy) { + set_cpu_online(smp_processor_id(), false); + prom_stopself(); } -/** - * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu - * @cpu: cpu to allocate for - * @size: size allocation in bytes - * @align: alignment - * - * Allocate @size bytes aligned at @align for cpu @cpu. This wrapper - * does the right thing for NUMA regardless of the current - * configuration. - * - * RETURNS: - * Pointer to the allocated area on success, NULL on failure. - */ -static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, - size_t align) +void smp_send_stop(void) { - const unsigned long goal = __pa(MAX_DMA_ADDRESS); -#ifdef CONFIG_NEED_MULTIPLE_NODES - int node = cpu_to_node(cpu); - void *ptr; - - if (!node_online(node) || !NODE_DATA(node)) { - ptr = __alloc_bootmem(size, align, goal); - pr_info("cpu %d has no node %d or node-local memory\n", - cpu, node); - pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n", - cpu, size, __pa(ptr)); - } else { - ptr = __alloc_bootmem_node(NODE_DATA(node), - size, align, goal); - pr_debug("per cpu data for cpu%d %lu bytes on node%d at " - "%016lx\n", cpu, size, node, __pa(ptr)); - } - return ptr; -#else - return __alloc_bootmem(size, align, goal); + int cpu; + + if (tlb_type == hypervisor) { + int this_cpu = smp_processor_id(); +#ifdef CONFIG_SERIAL_SUNHV + sunhv_migrate_hvcons_irq(this_cpu); #endif -} + for_each_online_cpu(cpu) { + if (cpu == this_cpu) + continue; -static void __init pcpu_free_bootmem(void *ptr, size_t size) -{ - free_bootmem(__pa(ptr), size); + set_cpu_online(cpu, false); +#ifdef CONFIG_SUN_LDOMS + if (ldom_domaining_enabled) { + unsigned long hv_err; + hv_err = sun4v_cpu_stop(cpu); + if (hv_err) + printk(KERN_ERR "sun4v_cpu_stop() " + "failed err=%lu\n", hv_err); + } else +#endif + prom_stopcpu_cpuid(cpu); + } + } else + smp_call_function(stop_this_cpu, NULL, 0); } static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) @@ -1467,27 +1524,9 @@ static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) return REMOTE_DISTANCE; } -static void __init pcpu_populate_pte(unsigned long addr) +static int __init pcpu_cpu_to_node(int cpu) { - pgd_t *pgd = pgd_offset_k(addr); - pud_t *pud; - pmd_t *pmd; - - pud = pud_offset(pgd, addr); - if (pud_none(*pud)) { - pmd_t *new; - - new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); - pud_populate(&init_mm, pud, new); - } - - pmd = pmd_offset(pud, addr); - if (!pmd_present(*pmd)) { - pte_t *new; - - new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); - pmd_populate_kernel(&init_mm, pmd, new); - } + return cpu_to_node(cpu); } void __init setup_per_cpu_areas(void) @@ -1500,18 +1539,15 @@ void __init setup_per_cpu_areas(void) rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, PERCPU_DYNAMIC_RESERVE, 4 << 20, pcpu_cpu_distance, - pcpu_alloc_bootmem, - pcpu_free_bootmem); + pcpu_cpu_to_node); if (rc) - pr_warning("PERCPU: %s allocator failed (%d), " - "falling back to page size\n", - pcpu_fc_names[pcpu_chosen_fc], rc); + pr_warn("PERCPU: %s allocator failed (%d), " + "falling back to page size\n", + pcpu_fc_names[pcpu_chosen_fc], rc); } if (rc < 0) rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, - pcpu_alloc_bootmem, - pcpu_free_bootmem, - pcpu_populate_pte); + pcpu_cpu_to_node); if (rc < 0) panic("cannot initialize percpu area (err=%d)", rc); diff --git a/arch/sparc/kernel/sparc_ksyms.c b/arch/sparc/kernel/sparc_ksyms.c new file mode 100644 index 000000000000..09aa69e422e5 --- /dev/null +++ b/arch/sparc/kernel/sparc_ksyms.c @@ -0,0 +1,12 @@ +/* + * arch/sparc/kernel/ksyms.c: Sparc specific ksyms support. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) + */ + +#include <linux/init.h> +#include <linux/export.h> + +/* This is needed only for drivers/sbus/char/openprom.c */ +EXPORT_SYMBOL(saved_command_line); diff --git a/arch/sparc/kernel/sparc_ksyms_32.c b/arch/sparc/kernel/sparc_ksyms_32.c deleted file mode 100644 index e521c54560f9..000000000000 --- a/arch/sparc/kernel/sparc_ksyms_32.c +++ /dev/null @@ -1,32 +0,0 @@ -/* - * arch/sparc/kernel/ksyms.c: Sparc specific ksyms support. - * - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) - */ - -#include <linux/module.h> -#include <linux/init.h> - -#include <asm/pgtable.h> -#include <asm/uaccess.h> -#include <asm/delay.h> -#include <asm/head.h> -#include <asm/dma.h> - -struct poll { - int fd; - short events; - short revents; -}; - -/* from entry.S */ -EXPORT_SYMBOL(__udelay); -EXPORT_SYMBOL(__ndelay); - -/* from head_32.S */ -EXPORT_SYMBOL(__ret_efault); -EXPORT_SYMBOL(empty_zero_page); - -/* Exporting a symbol from /init/main.c */ -EXPORT_SYMBOL(saved_command_line); diff --git a/arch/sparc/kernel/sparc_ksyms_64.c b/arch/sparc/kernel/sparc_ksyms_64.c deleted file mode 100644 index 9f5e24ddcc70..000000000000 --- a/arch/sparc/kernel/sparc_ksyms_64.c +++ /dev/null @@ -1,53 +0,0 @@ -/* arch/sparc64/kernel/sparc64_ksyms.c: Sparc64 specific ksyms support. - * - * Copyright (C) 1996, 2007 David S. Miller (davem@davemloft.net) - * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) - * Copyright (C) 1999 Jakub Jelinek (jj@ultra.linux.cz) - */ - -#include <linux/export.h> -#include <linux/pci.h> -#include <linux/init.h> -#include <linux/bitops.h> - -#include <asm/cpudata.h> -#include <asm/uaccess.h> -#include <asm/spitfire.h> -#include <asm/oplib.h> -#include <asm/hypervisor.h> -#include <asm/cacheflush.h> - -struct poll { - int fd; - short events; - short revents; -}; - -/* from helpers.S */ -EXPORT_SYMBOL(__flushw_user); -EXPORT_SYMBOL_GPL(real_hard_smp_processor_id); - -/* from head_64.S */ -EXPORT_SYMBOL(__ret_efault); -EXPORT_SYMBOL(tlb_type); -EXPORT_SYMBOL(sun4v_chip_type); -EXPORT_SYMBOL(prom_root_node); - -/* from hvcalls.S */ -EXPORT_SYMBOL(sun4v_niagara_getperf); -EXPORT_SYMBOL(sun4v_niagara_setperf); -EXPORT_SYMBOL(sun4v_niagara2_getperf); -EXPORT_SYMBOL(sun4v_niagara2_setperf); - -/* from hweight.S */ -EXPORT_SYMBOL(__arch_hweight8); -EXPORT_SYMBOL(__arch_hweight16); -EXPORT_SYMBOL(__arch_hweight32); -EXPORT_SYMBOL(__arch_hweight64); - -/* from ffs_ffz.S */ -EXPORT_SYMBOL(ffs); -EXPORT_SYMBOL(__ffs); - -/* Exporting a symbol from /init/main.c */ -EXPORT_SYMBOL(saved_command_line); diff --git a/arch/sparc/kernel/spiterrs.S b/arch/sparc/kernel/spiterrs.S index c357e40ffd01..5427af44099a 100644 --- a/arch/sparc/kernel/spiterrs.S +++ b/arch/sparc/kernel/spiterrs.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* We need to carefully read the error status, ACK the errors, * prevent recursive traps, and pass the information on to C * code for logging. @@ -85,7 +86,7 @@ __spitfire_cee_trap_continue: ba,pt %xcc, etraptl1 rd %pc, %g7 - ba,pt %xcc, 2f + ba,a,pt %xcc, 2f nop 1: ba,pt %xcc, etrap_irq @@ -100,8 +101,7 @@ __spitfire_cee_trap_continue: mov %l5, %o2 call spitfire_access_error add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size __spitfire_access_error,.-__spitfire_access_error /* This is the trap handler entry point for ECC correctable @@ -179,8 +179,7 @@ __spitfire_data_access_exception_tl1: mov %l5, %o2 call spitfire_data_access_exception_tl1 add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size __spitfire_data_access_exception_tl1,.-__spitfire_data_access_exception_tl1 .type __spitfire_data_access_exception,#function @@ -200,8 +199,7 @@ __spitfire_data_access_exception: mov %l5, %o2 call spitfire_data_access_exception add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size __spitfire_data_access_exception,.-__spitfire_data_access_exception .type __spitfire_insn_access_exception_tl1,#function @@ -220,8 +218,7 @@ __spitfire_insn_access_exception_tl1: mov %l5, %o2 call spitfire_insn_access_exception_tl1 add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size __spitfire_insn_access_exception_tl1,.-__spitfire_insn_access_exception_tl1 .type __spitfire_insn_access_exception,#function @@ -240,6 +237,5 @@ __spitfire_insn_access_exception: mov %l5, %o2 call spitfire_insn_access_exception add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap .size __spitfire_insn_access_exception,.-__spitfire_insn_access_exception diff --git a/arch/sparc/kernel/sstate.c b/arch/sparc/kernel/sstate.c index c59af546f522..3bcc4ddc6911 100644 --- a/arch/sparc/kernel/sstate.c +++ b/arch/sparc/kernel/sstate.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* sstate.c: System soft state support. * * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> @@ -5,6 +6,7 @@ #include <linux/kernel.h> #include <linux/notifier.h> +#include <linux/panic_notifier.h> #include <linux/reboot.h> #include <linux/init.h> @@ -43,8 +45,8 @@ static const char poweroff_msg[32] __attribute__((aligned(32))) = "Linux powering off"; static const char rebooting_msg[32] __attribute__((aligned(32))) = "Linux rebooting"; -static const char panicing_msg[32] __attribute__((aligned(32))) = - "Linux panicing"; +static const char panicking_msg[32] __attribute__((aligned(32))) = + "Linux panicking"; static int sstate_reboot_call(struct notifier_block *np, unsigned long type, void *_unused) { @@ -76,7 +78,7 @@ static struct notifier_block sstate_reboot_notifier = { static int sstate_panic_event(struct notifier_block *n, unsigned long event, void *ptr) { - do_set_sstate(HV_SOFT_STATE_TRANSITION, panicing_msg); + do_set_sstate(HV_SOFT_STATE_TRANSITION, panicking_msg); return NOTIFY_DONE; } diff --git a/arch/sparc/kernel/stacktrace.c b/arch/sparc/kernel/stacktrace.c index e78386a0029f..d8eb1d149f9f 100644 --- a/arch/sparc/kernel/stacktrace.c +++ b/arch/sparc/kernel/stacktrace.c @@ -1,4 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only #include <linux/sched.h> +#include <linux/sched/debug.h> #include <linux/stacktrace.h> #include <linux/thread_info.h> #include <linux/ftrace.h> @@ -56,9 +58,11 @@ static void __save_stack_trace(struct thread_info *tp, trace->entries[trace->nr_entries++] = pc; #ifdef CONFIG_FUNCTION_GRAPH_TRACER if ((pc + 8UL) == (unsigned long) &return_to_handler) { - int index = t->curr_ret_stack; - if (t->ret_stack && index >= graph) { - pc = t->ret_stack[index - graph].ret; + struct ftrace_ret_stack *ret_stack; + ret_stack = ftrace_graph_get_ret_stack(t, + graph); + if (ret_stack) { + pc = ret_stack->ret; if (trace->nr_entries < trace->max_entries) trace->entries[trace->nr_entries++] = pc; diff --git a/arch/sparc/kernel/starfire.c b/arch/sparc/kernel/starfire.c index 82281a566bb8..b8cd57d9182b 100644 --- a/arch/sparc/kernel/starfire.c +++ b/arch/sparc/kernel/starfire.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * starfire.c: Starfire/E10000 support. * @@ -28,11 +29,6 @@ void check_if_starfire(void) this_is_starfire = 1; } -int starfire_hard_smp_processor_id(void) -{ - return upa_readl(0x1fff40000d0UL); -} - /* * Each Starfire board has 32 registers which perform translation * and delivery of traditional interrupt packets into the extended diff --git a/arch/sparc/kernel/sun4d_irq.c b/arch/sparc/kernel/sun4d_irq.c index f8933be3ca8b..9a137c70e8d1 100644 --- a/arch/sparc/kernel/sun4d_irq.c +++ b/arch/sparc/kernel/sun4d_irq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * SS1000/SC2000 interrupt handling. * @@ -143,7 +144,7 @@ static void sun4d_sbus_handler_irq(int sbusl) } } -void sun4d_handler_irq(int pil, struct pt_regs *regs) +void sun4d_handler_irq(unsigned int pil, struct pt_regs *regs) { struct pt_regs *old_regs; /* SBUS IRQ level (1 - 7) */ @@ -188,7 +189,7 @@ void sun4d_handler_irq(int pil, struct pt_regs *regs) static void sun4d_mask_irq(struct irq_data *data) { - struct sun4d_handler_data *handler_data = data->handler_data; + struct sun4d_handler_data *handler_data = irq_data_get_irq_handler_data(data); unsigned int real_irq; #ifdef CONFIG_SMP int cpuid = handler_data->cpuid; @@ -206,7 +207,7 @@ static void sun4d_mask_irq(struct irq_data *data) static void sun4d_unmask_irq(struct irq_data *data) { - struct sun4d_handler_data *handler_data = data->handler_data; + struct sun4d_handler_data *handler_data = irq_data_get_irq_handler_data(data); unsigned int real_irq; #ifdef CONFIG_SMP int cpuid = handler_data->cpuid; @@ -236,7 +237,7 @@ static void sun4d_shutdown_irq(struct irq_data *data) irq_unlink(data->irq); } -struct irq_chip sun4d_irq = { +static struct irq_chip sun4d_irq = { .name = "sun4d", .irq_startup = sun4d_startup_irq, .irq_shutdown = sun4d_shutdown_irq, @@ -285,9 +286,9 @@ static void __init sun4d_load_profile_irqs(void) } } -unsigned int _sun4d_build_device_irq(unsigned int real_irq, - unsigned int pil, - unsigned int board) +static unsigned int _sun4d_build_device_irq(unsigned int real_irq, + unsigned int pil, + unsigned int board) { struct sun4d_handler_data *handler_data; unsigned int irq; @@ -320,8 +321,8 @@ err_out: -unsigned int sun4d_build_device_irq(struct platform_device *op, - unsigned int real_irq) +static unsigned int sun4d_build_device_irq(struct platform_device *op, + unsigned int real_irq) { struct device_node *dp = op->dev.of_node; struct device_node *board_parent, *bus = dp->parent; @@ -334,12 +335,12 @@ unsigned int sun4d_build_device_irq(struct platform_device *op, irq = real_irq; while (bus) { - if (!strcmp(bus->name, "sbi")) { + if (of_node_name_eq(bus, "sbi")) { bus_connection = "io-unit"; break; } - if (!strcmp(bus->name, "bootbus")) { + if (of_node_name_eq(bus, "bootbus")) { bus_connection = "cpu-unit"; break; } @@ -359,16 +360,16 @@ unsigned int sun4d_build_device_irq(struct platform_device *op, * If Bus nodes parent is not io-unit/cpu-unit or the io-unit/cpu-unit * lacks a "board#" property, something is very wrong. */ - if (!bus->parent || strcmp(bus->parent->name, bus_connection)) { - printk(KERN_ERR "%s: Error, parent is not %s.\n", - bus->full_name, bus_connection); + if (!of_node_name_eq(bus->parent, bus_connection)) { + printk(KERN_ERR "%pOF: Error, parent is not %s.\n", + bus, bus_connection); goto err_out; } board_parent = bus->parent; board = of_getintprop_default(board_parent, "board#", -1); if (board == -1) { - printk(KERN_ERR "%s: Error, lacks board# property.\n", - board_parent->full_name); + printk(KERN_ERR "%pOF: Error, lacks board# property.\n", + board_parent); goto err_out; } @@ -383,7 +384,8 @@ err_out: return irq; } -unsigned int sun4d_build_timer_irq(unsigned int board, unsigned int real_irq) +static unsigned int sun4d_build_timer_irq(unsigned int board, + unsigned int real_irq) { return _sun4d_build_device_irq(real_irq, real_irq, board); } diff --git a/arch/sparc/kernel/sun4d_smp.c b/arch/sparc/kernel/sun4d_smp.c index c9eb82f23d92..9a62a5cf3337 100644 --- a/arch/sparc/kernel/sun4d_smp.c +++ b/arch/sparc/kernel/sun4d_smp.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* Sparc SS1000/SC2000 SMP support. * * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) @@ -10,7 +11,7 @@ #include <linux/interrupt.h> #include <linux/profile.h> #include <linux/delay.h> -#include <linux/sched.h> +#include <linux/sched/mm.h> #include <linux/cpu.h> #include <asm/cacheflush.h> @@ -50,7 +51,7 @@ static inline void show_leds(int cpuid) "i" (ASI_M_CTL)); } -void __cpuinit sun4d_cpu_pre_starting(void *arg) +void sun4d_cpu_pre_starting(void *arg) { int cpuid = hard_smp_processor_id(); @@ -62,7 +63,7 @@ void __cpuinit sun4d_cpu_pre_starting(void *arg) cc_set_imsk((cc_get_imsk() & ~0x8000) | 0x4000); } -void __cpuinit sun4d_cpu_pre_online(void *arg) +void sun4d_cpu_pre_online(void *arg) { unsigned long flags; int cpuid; @@ -93,7 +94,7 @@ void __cpuinit sun4d_cpu_pre_online(void *arg) show_leds(cpuid); /* Attach to the address space of init_task. */ - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; local_ops->cache_all(); @@ -118,7 +119,7 @@ void __init smp4d_boot_cpus(void) local_ops->cache_all(); } -int __cpuinit smp4d_boot_one_cpu(int i, struct task_struct *idle) +int smp4d_boot_one_cpu(int i, struct task_struct *idle) { unsigned long *entry = &sun4d_cpu_startup; int timeout; @@ -204,7 +205,7 @@ static void __init smp4d_ipi_init(void) void sun4d_ipi_interrupt(void) { - struct sun4d_ipi_work *work = &__get_cpu_var(sun4d_ipi_work); + struct sun4d_ipi_work *work = this_cpu_ptr(&sun4d_ipi_work); if (work->single) { work->single = 0; @@ -267,7 +268,7 @@ static void sun4d_ipi_resched(int cpu) } static struct smp_funcall { - smpfunc_t func; + void *func; unsigned long arg1; unsigned long arg2; unsigned long arg3; @@ -280,7 +281,7 @@ static struct smp_funcall { static DEFINE_SPINLOCK(cross_call_lock); /* Cross calls must be serialized, at least currently. */ -static void sun4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, +static void sun4d_cross_call(void *func, cpumask_t mask, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4) { @@ -295,7 +296,7 @@ static void sun4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, * If you make changes here, make sure * gcc generates proper code... */ - register smpfunc_t f asm("i0") = func; + register void *f asm("i0") = func; register unsigned long a1 asm("i1") = arg1; register unsigned long a2 asm("i2") = arg2; register unsigned long a3 asm("i3") = arg3; @@ -352,11 +353,13 @@ static void sun4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, /* Running cross calls. */ void smp4d_cross_call_irq(void) { + void (*func)(unsigned long, unsigned long, unsigned long, unsigned long, + unsigned long) = ccall_info.func; int i = hard_smp_processor_id(); ccall_info.processors_in[i] = 1; - ccall_info.func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, - ccall_info.arg4, ccall_info.arg5); + func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, ccall_info.arg4, + ccall_info.arg5); ccall_info.processors_out[i] = 1; } diff --git a/arch/sparc/kernel/sun4m_irq.c b/arch/sparc/kernel/sun4m_irq.c index c5ade9d27a1d..1079638986b5 100644 --- a/arch/sparc/kernel/sun4m_irq.c +++ b/arch/sparc/kernel/sun4m_irq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * sun4m irq support * @@ -9,10 +10,12 @@ * Copyright (C) 1996 Dave Redman (djhr@tadpole.co.uk) */ +#include <linux/slab.h> +#include <linux/sched/debug.h> +#include <linux/pgtable.h> + #include <asm/timer.h> #include <asm/traps.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> #include <asm/irq.h> #include <asm/io.h> #include <asm/cacheflush.h> @@ -186,9 +189,10 @@ static unsigned long sun4m_imask[0x50] = { static void sun4m_mask_irq(struct irq_data *data) { - struct sun4m_handler_data *handler_data = data->handler_data; + struct sun4m_handler_data *handler_data; int cpu = smp_processor_id(); + handler_data = irq_data_get_irq_handler_data(data); if (handler_data->mask) { unsigned long flags; @@ -204,9 +208,10 @@ static void sun4m_mask_irq(struct irq_data *data) static void sun4m_unmask_irq(struct irq_data *data) { - struct sun4m_handler_data *handler_data = data->handler_data; + struct sun4m_handler_data *handler_data; int cpu = smp_processor_id(); + handler_data = irq_data_get_irq_handler_data(data); if (handler_data->mask) { unsigned long flags; diff --git a/arch/sparc/kernel/sun4m_smp.c b/arch/sparc/kernel/sun4m_smp.c index 8a65f158153d..056df034e79e 100644 --- a/arch/sparc/kernel/sun4m_smp.c +++ b/arch/sparc/kernel/sun4m_smp.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * sun4m SMP support. * @@ -8,7 +9,7 @@ #include <linux/interrupt.h> #include <linux/profile.h> #include <linux/delay.h> -#include <linux/sched.h> +#include <linux/sched/mm.h> #include <linux/cpu.h> #include <asm/cacheflush.h> @@ -34,11 +35,11 @@ swap_ulong(volatile unsigned long *ptr, unsigned long val) return val; } -void __cpuinit sun4m_cpu_pre_starting(void *arg) +void sun4m_cpu_pre_starting(void *arg) { } -void __cpuinit sun4m_cpu_pre_online(void *arg) +void sun4m_cpu_pre_online(void *arg) { int cpuid = hard_smp_processor_id(); @@ -59,7 +60,7 @@ void __cpuinit sun4m_cpu_pre_online(void *arg) : "memory" /* paranoid */); /* Attach to the address space of init_task. */ - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; while (!cpumask_test_cpu(cpuid, &smp_commenced_mask)) @@ -75,7 +76,7 @@ void __init smp4m_boot_cpus(void) local_ops->cache_all(); } -int __cpuinit smp4m_boot_one_cpu(int i, struct task_struct *idle) +int smp4m_boot_one_cpu(int i, struct task_struct *idle) { unsigned long *entry = &sun4m_cpu_startup; int timeout; @@ -156,7 +157,7 @@ static void sun4m_ipi_mask_one(int cpu) } static struct smp_funcall { - smpfunc_t func; + void *func; unsigned long arg1; unsigned long arg2; unsigned long arg3; @@ -169,7 +170,7 @@ static struct smp_funcall { static DEFINE_SPINLOCK(cross_call_lock); /* Cross calls must be serialized, at least currently. */ -static void sun4m_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, +static void sun4m_cross_call(void *func, cpumask_t mask, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4) { @@ -229,11 +230,13 @@ static void sun4m_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, /* Running cross calls. */ void smp4m_cross_call_irq(void) { + void (*func)(unsigned long, unsigned long, unsigned long, unsigned long, + unsigned long) = ccall_info.func; int i = smp_processor_id(); ccall_info.processors_in[i] = 1; - ccall_info.func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, - ccall_info.arg4, ccall_info.arg5); + func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, ccall_info.arg4, + ccall_info.arg5); ccall_info.processors_out[i] = 1; } @@ -247,7 +250,7 @@ void smp4m_percpu_timer_interrupt(struct pt_regs *regs) ce = &per_cpu(sparc32_clockevent, cpu); - if (ce->mode & CLOCK_EVT_MODE_PERIODIC) + if (clockevent_state_periodic(ce)) sun4m_clear_profile_irq(cpu); else sparc_config.load_profile_irq(cpu, 0); /* Is this needless? */ diff --git a/arch/sparc/kernel/sun4v_ivec.S b/arch/sparc/kernel/sun4v_ivec.S index 559bc5e9c199..6478ef4f6a15 100644 --- a/arch/sparc/kernel/sun4v_ivec.S +++ b/arch/sparc/kernel/sun4v_ivec.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* sun4v_ivec.S: Sun4v interrupt vector handling. * * Copyright (C) 2006 <davem@davemloft.net> @@ -26,6 +27,21 @@ sun4v_cpu_mondo: ldxa [%g0] ASI_SCRATCHPAD, %g4 sub %g4, TRAP_PER_CPU_FAULT_INFO, %g4 + /* Get smp_processor_id() into %g3 */ + sethi %hi(trap_block), %g5 + or %g5, %lo(trap_block), %g5 + sub %g4, %g5, %g3 + srlx %g3, TRAP_BLOCK_SZ_SHIFT, %g3 + + /* Increment cpu_mondo_counter[smp_processor_id()] */ + sethi %hi(cpu_mondo_counter), %g5 + or %g5, %lo(cpu_mondo_counter), %g5 + sllx %g3, 3, %g3 + add %g5, %g3, %g5 + ldx [%g5], %g3 + add %g3, 1, %g3 + stx %g3, [%g5] + /* Get CPU mondo queue base phys address into %g7. */ ldx [%g4 + TRAP_PER_CPU_CPU_MONDO_PA], %g7 diff --git a/arch/sparc/kernel/sun4v_mcd.S b/arch/sparc/kernel/sun4v_mcd.S new file mode 100644 index 000000000000..a419b7318406 --- /dev/null +++ b/arch/sparc/kernel/sun4v_mcd.S @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* sun4v_mcd.S: Sun4v memory corruption detected precise exception handler + * + * Copyright (c) 2015 Oracle and/or its affiliates. All rights reserved. + * Authors: Bob Picco <bob.picco@oracle.com>, + * Khalid Aziz <khalid.aziz@oracle.com> + */ + .text + .align 32 + +sun4v_mcd_detect_precise: + mov %l4, %o1 + mov %l5, %o2 + call sun4v_mem_corrupt_detect_precise + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + nop diff --git a/arch/sparc/kernel/sun4v_tlb_miss.S b/arch/sparc/kernel/sun4v_tlb_miss.S index bde867fd71e8..7ac9f3367674 100644 --- a/arch/sparc/kernel/sun4v_tlb_miss.S +++ b/arch/sparc/kernel/sun4v_tlb_miss.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* sun4v_tlb_miss.S: Sun4v TLB miss handlers. * * Copyright (C) 2006 <davem@davemloft.net> @@ -182,7 +183,7 @@ sun4v_tsb_miss_common: cmp %g5, -1 be,pt %xcc, 80f nop - COMPUTE_TSB_PTR(%g5, %g4, HPAGE_SHIFT, %g2, %g7) + COMPUTE_TSB_PTR(%g5, %g4, REAL_HPAGE_SHIFT, %g2, %g7) /* That clobbered %g2, reload it. */ ldxa [%g0] ASI_SCRATCHPAD, %g2 @@ -195,6 +196,11 @@ sun4v_tsb_miss_common: ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7 sun4v_itlb_error: + rdpr %tl, %g1 + cmp %g1, 1 + ble,pt %icc, sun4v_bad_ra + or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_ITLB, %g1 + sethi %hi(sun4v_err_itlb_vaddr), %g1 stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)] sethi %hi(sun4v_err_itlb_ctx), %g1 @@ -206,15 +212,10 @@ sun4v_itlb_error: sethi %hi(sun4v_err_itlb_error), %g1 stx %o0, [%g1 + %lo(sun4v_err_itlb_error)] + sethi %hi(1f), %g7 rdpr %tl, %g4 - cmp %g4, 1 - ble,pt %icc, 1f - sethi %hi(2f), %g7 ba,pt %xcc, etraptl1 - or %g7, %lo(2f), %g7 - -1: ba,pt %xcc, etrap -2: or %g7, %lo(2b), %g7 +1: or %g7, %lo(1f), %g7 mov %l4, %o1 call sun4v_itlb_error_report add %sp, PTREGS_OFF, %o0 @@ -222,6 +223,11 @@ sun4v_itlb_error: /* NOTREACHED */ sun4v_dtlb_error: + rdpr %tl, %g1 + cmp %g1, 1 + ble,pt %icc, sun4v_bad_ra + or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_DTLB, %g1 + sethi %hi(sun4v_err_dtlb_vaddr), %g1 stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)] sethi %hi(sun4v_err_dtlb_ctx), %g1 @@ -233,21 +239,23 @@ sun4v_dtlb_error: sethi %hi(sun4v_err_dtlb_error), %g1 stx %o0, [%g1 + %lo(sun4v_err_dtlb_error)] + sethi %hi(1f), %g7 rdpr %tl, %g4 - cmp %g4, 1 - ble,pt %icc, 1f - sethi %hi(2f), %g7 ba,pt %xcc, etraptl1 - or %g7, %lo(2f), %g7 - -1: ba,pt %xcc, etrap -2: or %g7, %lo(2b), %g7 +1: or %g7, %lo(1f), %g7 mov %l4, %o1 call sun4v_dtlb_error_report add %sp, PTREGS_OFF, %o0 /* NOTREACHED */ +sun4v_bad_ra: + or %g0, %g4, %g5 + ba,pt %xcc, sparc64_realfault_common + or %g1, %g0, %g4 + + /* NOTREACHED */ + /* Instruction Access Exception, tl0. */ sun4v_iacc: ldxa [%g0] ASI_SCRATCHPAD, %g2 @@ -345,6 +353,7 @@ sun4v_mna: call sun4v_do_mna add %sp, PTREGS_OFF, %o0 ba,a,pt %xcc, rtrap + nop /* Privileged Action. */ sun4v_privact: diff --git a/arch/sparc/kernel/sys32.S b/arch/sparc/kernel/sys32.S index f7c72b6efc27..a3d308f2043e 100644 --- a/arch/sparc/kernel/sys32.S +++ b/arch/sparc/kernel/sys32.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * sys32.S: I-cache tricks for 32-bit compatibility layer simple * conversions. @@ -12,266 +13,8 @@ .text -#define SIGN1(STUB,SYSCALL,REG1) \ - .align 32; \ - .globl STUB; \ -STUB: sethi %hi(SYSCALL), %g1; \ - jmpl %g1 + %lo(SYSCALL), %g0; \ - sra REG1, 0, REG1 - -#define SIGN2(STUB,SYSCALL,REG1,REG2) \ - .align 32; \ - .globl STUB; \ -STUB: sethi %hi(SYSCALL), %g1; \ - sra REG1, 0, REG1; \ - jmpl %g1 + %lo(SYSCALL), %g0; \ - sra REG2, 0, REG2 - -#define SIGN3(STUB,SYSCALL,REG1,REG2,REG3) \ - .align 32; \ - .globl STUB; \ -STUB: sra REG1, 0, REG1; \ - sethi %hi(SYSCALL), %g1; \ - sra REG2, 0, REG2; \ - jmpl %g1 + %lo(SYSCALL), %g0; \ - sra REG3, 0, REG3 - -SIGN1(sys32_readahead, compat_sys_readahead, %o0) -SIGN2(sys32_fadvise64, compat_sys_fadvise64, %o0, %o4) -SIGN2(sys32_fadvise64_64, compat_sys_fadvise64_64, %o0, %o5) -SIGN1(sys32_clock_nanosleep, compat_sys_clock_nanosleep, %o1) -SIGN1(sys32_timer_settime, compat_sys_timer_settime, %o1) -SIGN1(sys32_io_submit, compat_sys_io_submit, %o1) -SIGN1(sys32_mq_open, compat_sys_mq_open, %o1) -SIGN1(sys32_select, compat_sys_select, %o0) -SIGN3(sys32_futex, compat_sys_futex, %o1, %o2, %o5) -SIGN1(sys32_recvfrom, compat_sys_recvfrom, %o0) -SIGN1(sys32_recvmsg, compat_sys_recvmsg, %o0) -SIGN1(sys32_sendmsg, compat_sys_sendmsg, %o0) - .globl sys32_mmap2 sys32_mmap2: sethi %hi(sys_mmap), %g1 jmpl %g1 + %lo(sys_mmap), %g0 sllx %o5, 12, %o5 - - .align 32 - .globl sys32_socketcall -sys32_socketcall: /* %o0=call, %o1=args */ - cmp %o0, 1 - bl,pn %xcc, do_einval - cmp %o0, 18 - bg,pn %xcc, do_einval - sub %o0, 1, %o0 - sllx %o0, 5, %o0 - sethi %hi(__socketcall_table_begin), %g2 - or %g2, %lo(__socketcall_table_begin), %g2 - jmpl %g2 + %o0, %g0 - nop -do_einval: - retl - mov -EINVAL, %o0 - - .align 32 -__socketcall_table_begin: - - /* Each entry is exactly 32 bytes. */ -do_sys_socket: /* sys_socket(int, int, int) */ -1: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_socket), %g1 -2: ldswa [%o1 + 0x8] %asi, %o2 - jmpl %g1 + %lo(sys_socket), %g0 -3: ldswa [%o1 + 0x4] %asi, %o1 - nop - nop - nop -do_sys_bind: /* sys_bind(int fd, struct sockaddr *, int) */ -4: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_bind), %g1 -5: ldswa [%o1 + 0x8] %asi, %o2 - jmpl %g1 + %lo(sys_bind), %g0 -6: lduwa [%o1 + 0x4] %asi, %o1 - nop - nop - nop -do_sys_connect: /* sys_connect(int, struct sockaddr *, int) */ -7: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_connect), %g1 -8: ldswa [%o1 + 0x8] %asi, %o2 - jmpl %g1 + %lo(sys_connect), %g0 -9: lduwa [%o1 + 0x4] %asi, %o1 - nop - nop - nop -do_sys_listen: /* sys_listen(int, int) */ -10: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_listen), %g1 - jmpl %g1 + %lo(sys_listen), %g0 -11: ldswa [%o1 + 0x4] %asi, %o1 - nop - nop - nop - nop -do_sys_accept: /* sys_accept(int, struct sockaddr *, int *) */ -12: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_accept), %g1 -13: lduwa [%o1 + 0x8] %asi, %o2 - jmpl %g1 + %lo(sys_accept), %g0 -14: lduwa [%o1 + 0x4] %asi, %o1 - nop - nop - nop -do_sys_getsockname: /* sys_getsockname(int, struct sockaddr *, int *) */ -15: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_getsockname), %g1 -16: lduwa [%o1 + 0x8] %asi, %o2 - jmpl %g1 + %lo(sys_getsockname), %g0 -17: lduwa [%o1 + 0x4] %asi, %o1 - nop - nop - nop -do_sys_getpeername: /* sys_getpeername(int, struct sockaddr *, int *) */ -18: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_getpeername), %g1 -19: lduwa [%o1 + 0x8] %asi, %o2 - jmpl %g1 + %lo(sys_getpeername), %g0 -20: lduwa [%o1 + 0x4] %asi, %o1 - nop - nop - nop -do_sys_socketpair: /* sys_socketpair(int, int, int, int *) */ -21: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_socketpair), %g1 -22: ldswa [%o1 + 0x8] %asi, %o2 -23: lduwa [%o1 + 0xc] %asi, %o3 - jmpl %g1 + %lo(sys_socketpair), %g0 -24: ldswa [%o1 + 0x4] %asi, %o1 - nop - nop -do_sys_send: /* sys_send(int, void *, size_t, unsigned int) */ -25: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_send), %g1 -26: lduwa [%o1 + 0x8] %asi, %o2 -27: lduwa [%o1 + 0xc] %asi, %o3 - jmpl %g1 + %lo(sys_send), %g0 -28: lduwa [%o1 + 0x4] %asi, %o1 - nop - nop -do_sys_recv: /* sys_recv(int, void *, size_t, unsigned int) */ -29: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_recv), %g1 -30: lduwa [%o1 + 0x8] %asi, %o2 -31: lduwa [%o1 + 0xc] %asi, %o3 - jmpl %g1 + %lo(sys_recv), %g0 -32: lduwa [%o1 + 0x4] %asi, %o1 - nop - nop -do_sys_sendto: /* sys_sendto(int, u32, compat_size_t, unsigned int, u32, int) */ -33: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_sendto), %g1 -34: lduwa [%o1 + 0x8] %asi, %o2 -35: lduwa [%o1 + 0xc] %asi, %o3 -36: lduwa [%o1 + 0x10] %asi, %o4 -37: ldswa [%o1 + 0x14] %asi, %o5 - jmpl %g1 + %lo(sys_sendto), %g0 -38: lduwa [%o1 + 0x4] %asi, %o1 -do_sys_recvfrom: /* sys_recvfrom(int, u32, compat_size_t, unsigned int, u32, u32) */ -39: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_recvfrom), %g1 -40: lduwa [%o1 + 0x8] %asi, %o2 -41: lduwa [%o1 + 0xc] %asi, %o3 -42: lduwa [%o1 + 0x10] %asi, %o4 -43: lduwa [%o1 + 0x14] %asi, %o5 - jmpl %g1 + %lo(sys_recvfrom), %g0 -44: lduwa [%o1 + 0x4] %asi, %o1 -do_sys_shutdown: /* sys_shutdown(int, int) */ -45: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_shutdown), %g1 - jmpl %g1 + %lo(sys_shutdown), %g0 -46: ldswa [%o1 + 0x4] %asi, %o1 - nop - nop - nop - nop -do_sys_setsockopt: /* compat_sys_setsockopt(int, int, int, char *, int) */ -47: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(compat_sys_setsockopt), %g1 -48: ldswa [%o1 + 0x8] %asi, %o2 -49: lduwa [%o1 + 0xc] %asi, %o3 -50: ldswa [%o1 + 0x10] %asi, %o4 - jmpl %g1 + %lo(compat_sys_setsockopt), %g0 -51: ldswa [%o1 + 0x4] %asi, %o1 - nop -do_sys_getsockopt: /* compat_sys_getsockopt(int, int, int, u32, u32) */ -52: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(compat_sys_getsockopt), %g1 -53: ldswa [%o1 + 0x8] %asi, %o2 -54: lduwa [%o1 + 0xc] %asi, %o3 -55: lduwa [%o1 + 0x10] %asi, %o4 - jmpl %g1 + %lo(compat_sys_getsockopt), %g0 -56: ldswa [%o1 + 0x4] %asi, %o1 - nop -do_sys_sendmsg: /* compat_sys_sendmsg(int, struct compat_msghdr *, unsigned int) */ -57: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(compat_sys_sendmsg), %g1 -58: lduwa [%o1 + 0x8] %asi, %o2 - jmpl %g1 + %lo(compat_sys_sendmsg), %g0 -59: lduwa [%o1 + 0x4] %asi, %o1 - nop - nop - nop -do_sys_recvmsg: /* compat_sys_recvmsg(int, struct compat_msghdr *, unsigned int) */ -60: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(compat_sys_recvmsg), %g1 -61: lduwa [%o1 + 0x8] %asi, %o2 - jmpl %g1 + %lo(compat_sys_recvmsg), %g0 -62: lduwa [%o1 + 0x4] %asi, %o1 - nop - nop - nop -do_sys_accept4: /* sys_accept4(int, struct sockaddr *, int *, int) */ -63: ldswa [%o1 + 0x0] %asi, %o0 - sethi %hi(sys_accept4), %g1 -64: lduwa [%o1 + 0x8] %asi, %o2 -65: ldswa [%o1 + 0xc] %asi, %o3 - jmpl %g1 + %lo(sys_accept4), %g0 -66: lduwa [%o1 + 0x4] %asi, %o1 - nop - nop - - .section __ex_table,"a" - .align 4 - .word 1b, __retl_efault, 2b, __retl_efault - .word 3b, __retl_efault, 4b, __retl_efault - .word 5b, __retl_efault, 6b, __retl_efault - .word 7b, __retl_efault, 8b, __retl_efault - .word 9b, __retl_efault, 10b, __retl_efault - .word 11b, __retl_efault, 12b, __retl_efault - .word 13b, __retl_efault, 14b, __retl_efault - .word 15b, __retl_efault, 16b, __retl_efault - .word 17b, __retl_efault, 18b, __retl_efault - .word 19b, __retl_efault, 20b, __retl_efault - .word 21b, __retl_efault, 22b, __retl_efault - .word 23b, __retl_efault, 24b, __retl_efault - .word 25b, __retl_efault, 26b, __retl_efault - .word 27b, __retl_efault, 28b, __retl_efault - .word 29b, __retl_efault, 30b, __retl_efault - .word 31b, __retl_efault, 32b, __retl_efault - .word 33b, __retl_efault, 34b, __retl_efault - .word 35b, __retl_efault, 36b, __retl_efault - .word 37b, __retl_efault, 38b, __retl_efault - .word 39b, __retl_efault, 40b, __retl_efault - .word 41b, __retl_efault, 42b, __retl_efault - .word 43b, __retl_efault, 44b, __retl_efault - .word 45b, __retl_efault, 46b, __retl_efault - .word 47b, __retl_efault, 48b, __retl_efault - .word 49b, __retl_efault, 50b, __retl_efault - .word 51b, __retl_efault, 52b, __retl_efault - .word 53b, __retl_efault, 54b, __retl_efault - .word 55b, __retl_efault, 56b, __retl_efault - .word 57b, __retl_efault, 58b, __retl_efault - .word 59b, __retl_efault, 60b, __retl_efault - .word 61b, __retl_efault, 62b, __retl_efault - .word 63b, __retl_efault, 64b, __retl_efault - .word 65b, __retl_efault, 66b, __retl_efault - .previous diff --git a/arch/sparc/kernel/sys_sparc32.c b/arch/sparc/kernel/sys_sparc32.c index 3d0ddbc005fe..f84a02ab6bf9 100644 --- a/arch/sparc/kernel/sys_sparc32.c +++ b/arch/sparc/kernel/sys_sparc32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* sys_sparc32.c: Conversion between 32bit and 64bit native syscalls. * * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) @@ -21,7 +22,6 @@ #include <linux/msg.h> #include <linux/shm.h> #include <linux/uio.h> -#include <linux/nfs_fs.h> #include <linux/quota.h> #include <linux/poll.h> #include <linux/personality.h> @@ -44,25 +44,21 @@ #include <linux/slab.h> #include <asm/types.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/fpumacro.h> #include <asm/mmu_context.h> #include <asm/compat_signal.h> -asmlinkage long sys32_truncate64(const char __user * path, unsigned long high, unsigned long low) +#include "systbls.h" + +COMPAT_SYSCALL_DEFINE3(truncate64, const char __user *, path, u32, high, u32, low) { - if ((int)high < 0) - return -EINVAL; - else - return sys_truncate(path, (high << 32) | low); + return ksys_truncate(path, ((u64)high << 32) | low); } -asmlinkage long sys32_ftruncate64(unsigned int fd, unsigned long high, unsigned long low) +COMPAT_SYSCALL_DEFINE3(ftruncate64, unsigned int, fd, u32, high, u32, low) { - if ((int)high < 0) - return -EINVAL; - else - return sys_ftruncate(fd, (high << 32) | low); + return ksys_ftruncate(fd, ((u64)high << 32) | low); } static int cp_compat_stat64(struct kstat *stat, @@ -95,8 +91,8 @@ static int cp_compat_stat64(struct kstat *stat, return err; } -asmlinkage long compat_sys_stat64(const char __user * filename, - struct compat_stat64 __user *statbuf) +COMPAT_SYSCALL_DEFINE2(stat64, const char __user *, filename, + struct compat_stat64 __user *, statbuf) { struct kstat stat; int error = vfs_stat(filename, &stat); @@ -106,8 +102,8 @@ asmlinkage long compat_sys_stat64(const char __user * filename, return error; } -asmlinkage long compat_sys_lstat64(const char __user * filename, - struct compat_stat64 __user *statbuf) +COMPAT_SYSCALL_DEFINE2(lstat64, const char __user *, filename, + struct compat_stat64 __user *, statbuf) { struct kstat stat; int error = vfs_lstat(filename, &stat); @@ -117,8 +113,8 @@ asmlinkage long compat_sys_lstat64(const char __user * filename, return error; } -asmlinkage long compat_sys_fstat64(unsigned int fd, - struct compat_stat64 __user * statbuf) +COMPAT_SYSCALL_DEFINE2(fstat64, unsigned int, fd, + struct compat_stat64 __user *, statbuf) { struct kstat stat; int error = vfs_fstat(fd, &stat); @@ -128,9 +124,9 @@ asmlinkage long compat_sys_fstat64(unsigned int fd, return error; } -asmlinkage long compat_sys_fstatat64(unsigned int dfd, - const char __user *filename, - struct compat_stat64 __user * statbuf, int flag) +COMPAT_SYSCALL_DEFINE4(fstatat64, unsigned int, dfd, + const char __user *, filename, + struct compat_stat64 __user *, statbuf, int, flag) { struct kstat stat; int error; @@ -157,7 +153,6 @@ COMPAT_SYSCALL_DEFINE5(rt_sigaction, int, sig, { struct k_sigaction new_ka, old_ka; int ret; - compat_sigset_t set32; /* XXX: Don't preclude handling different sized sigset_t's. */ if (sigsetsize != sizeof(compat_sigset_t)) @@ -169,10 +164,9 @@ COMPAT_SYSCALL_DEFINE5(rt_sigaction, int, sig, new_ka.ka_restorer = restorer; ret = get_user(u_handler, &act->sa_handler); new_ka.sa.sa_handler = compat_ptr(u_handler); - ret |= __copy_from_user(&set32, &act->sa_mask, sizeof(compat_sigset_t)); - sigset_from_compat(&new_ka.sa.sa_mask, &set32); - ret |= __get_user(new_ka.sa.sa_flags, &act->sa_flags); - ret |= __get_user(u_restorer, &act->sa_restorer); + ret |= get_compat_sigset(&new_ka.sa.sa_mask, &act->sa_mask); + ret |= get_user(new_ka.sa.sa_flags, &act->sa_flags); + ret |= get_user(u_restorer, &act->sa_restorer); new_ka.sa.sa_restorer = compat_ptr(u_restorer); if (ret) return -EFAULT; @@ -181,11 +175,11 @@ COMPAT_SYSCALL_DEFINE5(rt_sigaction, int, sig, ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); if (!ret && oact) { - sigset_to_compat(&set32, &old_ka.sa.sa_mask); ret = put_user(ptr_to_compat(old_ka.sa.sa_handler), &oact->sa_handler); - ret |= __copy_to_user(&oact->sa_mask, &set32, sizeof(compat_sigset_t)); - ret |= __put_user(old_ka.sa.sa_flags, &oact->sa_flags); - ret |= __put_user(ptr_to_compat(old_ka.sa.sa_restorer), &oact->sa_restorer); + ret |= put_compat_sigset(&oact->sa_mask, &old_ka.sa.sa_mask, + sizeof(oact->sa_mask)); + ret |= put_user(old_ka.sa.sa_flags, &oact->sa_flags); + ret |= put_user(ptr_to_compat(old_ka.sa.sa_restorer), &oact->sa_restorer); if (ret) ret = -EFAULT; } @@ -193,62 +187,51 @@ COMPAT_SYSCALL_DEFINE5(rt_sigaction, int, sig, return ret; } -asmlinkage compat_ssize_t sys32_pread64(unsigned int fd, - char __user *ubuf, - compat_size_t count, - unsigned long poshi, - unsigned long poslo) +COMPAT_SYSCALL_DEFINE5(pread64, unsigned int, fd, char __user *, ubuf, + compat_size_t, count, u32, poshi, u32, poslo) { - return sys_pread64(fd, ubuf, count, (poshi << 32) | poslo); + return ksys_pread64(fd, ubuf, count, ((u64)poshi << 32) | poslo); } -asmlinkage compat_ssize_t sys32_pwrite64(unsigned int fd, - char __user *ubuf, - compat_size_t count, - unsigned long poshi, - unsigned long poslo) +COMPAT_SYSCALL_DEFINE5(pwrite64, unsigned int, fd, char __user *, ubuf, + compat_size_t, count, u32, poshi, u32, poslo) { - return sys_pwrite64(fd, ubuf, count, (poshi << 32) | poslo); + return ksys_pwrite64(fd, ubuf, count, ((u64)poshi << 32) | poslo); } -asmlinkage long compat_sys_readahead(int fd, - unsigned long offhi, - unsigned long offlo, - compat_size_t count) +COMPAT_SYSCALL_DEFINE4(readahead, int, fd, u32, offhi, u32, offlo, + compat_size_t, count) { - return sys_readahead(fd, (offhi << 32) | offlo, count); + return ksys_readahead(fd, ((u64)offhi << 32) | offlo, count); } -long compat_sys_fadvise64(int fd, - unsigned long offhi, - unsigned long offlo, - compat_size_t len, int advice) +COMPAT_SYSCALL_DEFINE5(fadvise64, int, fd, u32, offhi, u32, offlo, + compat_size_t, len, int, advice) { - return sys_fadvise64_64(fd, (offhi << 32) | offlo, len, advice); + return ksys_fadvise64_64(fd, ((u64)offhi << 32) | offlo, len, advice); } -long compat_sys_fadvise64_64(int fd, - unsigned long offhi, unsigned long offlo, - unsigned long lenhi, unsigned long lenlo, - int advice) +COMPAT_SYSCALL_DEFINE6(fadvise64_64, int, fd, u32, offhi, u32, offlo, + u32, lenhi, u32, lenlo, int, advice) { - return sys_fadvise64_64(fd, - (offhi << 32) | offlo, - (lenhi << 32) | lenlo, - advice); + return ksys_fadvise64_64(fd, + ((u64)offhi << 32) | offlo, + ((u64)lenhi << 32) | lenlo, + advice); } -long sys32_sync_file_range(unsigned int fd, unsigned long off_high, unsigned long off_low, unsigned long nb_high, unsigned long nb_low, unsigned int flags) +COMPAT_SYSCALL_DEFINE6(sync_file_range, unsigned int, fd, u32, off_high, u32, off_low, + u32, nb_high, u32, nb_low, unsigned int, flags) { - return sys_sync_file_range(fd, - (off_high << 32) | off_low, - (nb_high << 32) | nb_low, - flags); + return ksys_sync_file_range(fd, + ((u64)off_high << 32) | off_low, + ((u64)nb_high << 32) | nb_low, + flags); } -asmlinkage long compat_sys_fallocate(int fd, int mode, u32 offhi, u32 offlo, - u32 lenhi, u32 lenlo) +COMPAT_SYSCALL_DEFINE6(fallocate, int, fd, int, mode, u32, offhi, u32, offlo, + u32, lenhi, u32, lenlo) { - return sys_fallocate(fd, mode, ((loff_t)offhi << 32) | offlo, - ((loff_t)lenhi << 32) | lenlo); + return ksys_fallocate(fd, mode, ((loff_t)offhi << 32) | offlo, + ((loff_t)lenhi << 32) | lenlo); } diff --git a/arch/sparc/kernel/sys_sparc_32.c b/arch/sparc/kernel/sys_sparc_32.c index 3a8d1844402e..fb31bc0c5b48 100644 --- a/arch/sparc/kernel/sys_sparc_32.c +++ b/arch/sparc/kernel/sys_sparc_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* linux/arch/sparc/kernel/sys_sparc.c * * This file contains various random system calls that @@ -7,7 +8,9 @@ #include <linux/errno.h> #include <linux/types.h> -#include <linux/sched.h> +#include <linux/sched/signal.h> +#include <linux/sched/mm.h> +#include <linux/sched/debug.h> #include <linux/mm.h> #include <linux/fs.h> #include <linux/file.h> @@ -20,29 +23,36 @@ #include <linux/utsname.h> #include <linux/smp.h> #include <linux/ipc.h> +#include <linux/hugetlb.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/unistd.h> +#include "systbls.h" + /* #define DEBUG_UNIMP_SYSCALL */ /* XXX Make this per-binary type, this way we can detect the type of * XXX a binary. Every Sparc executable calls this very early on. */ -asmlinkage unsigned long sys_getpagesize(void) +SYSCALL_DEFINE0(getpagesize) { return PAGE_SIZE; /* Possibly older binaries want 8192 on sun4's? */ } -unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) +unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags) { - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; + bool file_hugepage = false; + + if (filp && is_file_hugepages(filp)) + file_hugepage = true; if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate * cache aliasing constraints. */ - if ((flags & MAP_SHARED) && + if (!file_hugepage && (flags & MAP_SHARED) && ((addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1))) return -EINVAL; return addr; @@ -54,13 +64,16 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi if (!addr) addr = TASK_UNMAPPED_BASE; - info.flags = 0; info.length = len; info.low_limit = addr; info.high_limit = TASK_SIZE; - info.align_mask = (flags & MAP_SHARED) ? - (PAGE_MASK & (SHMLBA - 1)) : 0; - info.align_offset = pgoff << PAGE_SHIFT; + if (!file_hugepage) { + info.align_mask = (flags & MAP_SHARED) ? + (PAGE_MASK & (SHMLBA - 1)) : 0; + info.align_offset = pgoff << PAGE_SHIFT; + } else { + info.align_mask = huge_page_mask_align(filp); + } return vm_unmapped_area(&info); } @@ -68,7 +81,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi * sys_pipe() is the normal C calling standard for creating * a pipe. It's not the way unix traditionally does this, though. */ -asmlinkage int sparc_pipe(struct pt_regs *regs) +SYSCALL_DEFINE0(sparc_pipe) { int fd[2]; int error; @@ -76,7 +89,7 @@ asmlinkage int sparc_pipe(struct pt_regs *regs) error = do_pipe_flags(fd, 0); if (error) goto out; - regs->u_regs[UREG_I1] = fd[1]; + current_pt_regs()->u_regs[UREG_I1] = fd[1]; error = fd[0]; out: return error; @@ -93,27 +106,27 @@ int sparc_mmap_check(unsigned long addr, unsigned long len) /* Linux version of mmap */ -asmlinkage unsigned long sys_mmap2(unsigned long addr, unsigned long len, - unsigned long prot, unsigned long flags, unsigned long fd, - unsigned long pgoff) +SYSCALL_DEFINE6(mmap2, unsigned long, addr, unsigned long, len, + unsigned long, prot, unsigned long, flags, unsigned long, fd, + unsigned long, pgoff) { /* Make sure the shift for mmap2 is constant (12), no matter what PAGE_SIZE we have. */ - return sys_mmap_pgoff(addr, len, prot, flags, fd, - pgoff >> (PAGE_SHIFT - 12)); + return ksys_mmap_pgoff(addr, len, prot, flags, fd, + pgoff >> (PAGE_SHIFT - 12)); } -asmlinkage unsigned long sys_mmap(unsigned long addr, unsigned long len, - unsigned long prot, unsigned long flags, unsigned long fd, - unsigned long off) +SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, + unsigned long, prot, unsigned long, flags, unsigned long, fd, + unsigned long, off) { /* no alignment check? */ - return sys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); + return ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); } -long sparc_remap_file_pages(unsigned long start, unsigned long size, - unsigned long prot, unsigned long pgoff, - unsigned long flags) +SYSCALL_DEFINE5(sparc_remap_file_pages, unsigned long, start, unsigned long, size, + unsigned long, prot, unsigned long, pgoff, + unsigned long, flags) { /* This works on an existing mmap so we don't need to validate * the range as that was done at the original mmap call. @@ -122,11 +135,10 @@ long sparc_remap_file_pages(unsigned long start, unsigned long size, (pgoff >> (PAGE_SHIFT - 12)), flags); } -/* we come to here via sys_nis_syscall so it can setup the regs argument */ -asmlinkage unsigned long -c_sys_nis_syscall (struct pt_regs *regs) +SYSCALL_DEFINE0(nis_syscall) { static int count = 0; + struct pt_regs *regs = current_pt_regs(); if (count++ > 5) return -ENOSYS; @@ -143,17 +155,11 @@ c_sys_nis_syscall (struct pt_regs *regs) asmlinkage void sparc_breakpoint (struct pt_regs *regs) { - siginfo_t info; #ifdef DEBUG_SPARC_BREAKPOINT printk ("TRAP: Entering kernel PC=%x, nPC=%x\n", regs->pc, regs->npc); #endif - info.si_signo = SIGTRAP; - info.si_errno = 0; - info.si_code = TRAP_BRKPT; - info.si_addr = (void __user *)regs->pc; - info.si_trapno = 0; - force_sig_info(SIGTRAP, &info, current); + force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->pc); #ifdef DEBUG_SPARC_BREAKPOINT printk ("TRAP: Returning to space: PC=%x nPC=%x\n", regs->pc, regs->npc); @@ -197,25 +203,29 @@ SYSCALL_DEFINE5(rt_sigaction, int, sig, return ret; } -asmlinkage int sys_getdomainname(char __user *name, int len) +SYSCALL_DEFINE2(getdomainname, char __user *, name, int, len) { - int nlen, err; - + int nlen, err; + char tmp[__NEW_UTS_LEN + 1]; + if (len < 0) return -EINVAL; - down_read(&uts_sem); - + down_read(&uts_sem); + nlen = strlen(utsname()->domainname) + 1; err = -EINVAL; if (nlen > len) - goto out; + goto out_unlock; + memcpy(tmp, utsname()->domainname, nlen); - err = -EFAULT; - if (!copy_to_user(name, utsname()->domainname, nlen)) - err = 0; + up_read(&uts_sem); -out: + if (copy_to_user(name, tmp, nlen)) + return -EFAULT; + return 0; + +out_unlock: up_read(&uts_sem); return err; } diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c index 51561b8b15ba..dbf118b40601 100644 --- a/arch/sparc/kernel/sys_sparc_64.c +++ b/arch/sparc/kernel/sys_sparc_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* linux/arch/sparc64/kernel/sys_sparc.c * * This file contains various random system calls that @@ -7,7 +8,9 @@ #include <linux/errno.h> #include <linux/types.h> -#include <linux/sched.h> +#include <linux/sched/signal.h> +#include <linux/sched/mm.h> +#include <linux/sched/debug.h> #include <linux/fs.h> #include <linux/file.h> #include <linux/mm.h> @@ -24,24 +27,25 @@ #include <linux/personality.h> #include <linux/random.h> #include <linux/export.h> +#include <linux/context_tracking.h> +#include <linux/timex.h> +#include <linux/uaccess.h> +#include <linux/hugetlb.h> -#include <asm/uaccess.h> #include <asm/utrap.h> #include <asm/unistd.h> #include "entry.h" +#include "kernel.h" #include "systbls.h" /* #define DEBUG_UNIMP_SYSCALL */ -asmlinkage unsigned long sys_getpagesize(void) +SYSCALL_DEFINE0(getpagesize) { return PAGE_SIZE; } -#define VA_EXCLUDE_START (0x0000080000000000UL - (1UL << 32UL)) -#define VA_EXCLUDE_END (0xfffff80000000000UL + (1UL << 32UL)) - /* Does addr --> addr+len fall within 4GB of the VA-space hole or * overflow past the end of the 64-bit address space? */ @@ -84,19 +88,33 @@ static inline unsigned long COLOR_ALIGN(unsigned long addr, return base + off; } -unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) +static unsigned long get_align_mask(struct file *filp, unsigned long flags) +{ + if (filp && is_file_hugepages(filp)) + return huge_page_mask_align(filp); + if (filp || (flags & MAP_SHARED)) + return PAGE_MASK & (SHMLBA - 1); + + return 0; +} + +unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags) { struct mm_struct *mm = current->mm; struct vm_area_struct * vma; unsigned long task_size = TASK_SIZE; int do_color_align; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; + bool file_hugepage = false; + + if (filp && is_file_hugepages(filp)) + file_hugepage = true; if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate * cache aliasing constraints. */ - if ((flags & MAP_SHARED) && + if (!file_hugepage && (flags & MAP_SHARED) && ((addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1))) return -EINVAL; return addr; @@ -108,7 +126,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi return -ENOMEM; do_color_align = 0; - if (filp || (flags & MAP_SHARED)) + if ((filp || (flags & MAP_SHARED)) && !file_hugepage) do_color_align = 1; if (addr) { @@ -119,16 +137,16 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi vma = find_vma(mm, addr); if (task_size - len >= addr && - (!vma || addr + len <= vma->vm_start)) + (!vma || addr + len <= vm_start_gap(vma))) return addr; } - info.flags = 0; info.length = len; info.low_limit = TASK_UNMAPPED_BASE; info.high_limit = min(task_size, VA_EXCLUDE_START); - info.align_mask = do_color_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; - info.align_offset = pgoff << PAGE_SHIFT; + info.align_mask = get_align_mask(filp, flags); + if (!file_hugepage) + info.align_offset = pgoff << PAGE_SHIFT; addr = vm_unmapped_area(&info); if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) { @@ -144,23 +162,27 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi unsigned long arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, const unsigned long len, const unsigned long pgoff, - const unsigned long flags) + const unsigned long flags, vm_flags_t vm_flags) { struct vm_area_struct *vma; struct mm_struct *mm = current->mm; unsigned long task_size = STACK_TOP32; unsigned long addr = addr0; int do_color_align; - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; + bool file_hugepage = false; /* This should only ever run for 32-bit processes. */ BUG_ON(!test_thread_flag(TIF_32BIT)); + if (filp && is_file_hugepages(filp)) + file_hugepage = true; + if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate * cache aliasing constraints. */ - if ((flags & MAP_SHARED) && + if (!file_hugepage && (flags & MAP_SHARED) && ((addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1))) return -EINVAL; return addr; @@ -170,7 +192,7 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, return -ENOMEM; do_color_align = 0; - if (filp || (flags & MAP_SHARED)) + if ((filp || (flags & MAP_SHARED)) && !file_hugepage) do_color_align = 1; /* requesting a specific address */ @@ -182,7 +204,7 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, vma = find_vma(mm, addr); if (task_size - len >= addr && - (!vma || addr + len <= vma->vm_start)) + (!vma || addr + len <= vm_start_gap(vma))) return addr; } @@ -190,8 +212,9 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; - info.align_mask = do_color_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; - info.align_offset = pgoff << PAGE_SHIFT; + info.align_mask = get_align_mask(filp, flags); + if (!file_hugepage) + info.align_offset = pgoff << PAGE_SHIFT; addr = vm_unmapped_area(&info); /* @@ -215,14 +238,10 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, unsigned long get_fb_unmapped_area(struct file *filp, unsigned long orig_addr, unsigned long len, unsigned long pgoff, unsigned long flags) { unsigned long align_goal, addr = -ENOMEM; - unsigned long (*get_area)(struct file *, unsigned long, - unsigned long, unsigned long, unsigned long); - - get_area = current->mm->get_unmapped_area; if (flags & MAP_FIXED) { /* Ok, don't mess with it. */ - return get_area(NULL, orig_addr, len, pgoff, flags); + return mm_get_unmapped_area(NULL, orig_addr, len, pgoff, flags); } flags &= ~MAP_SHARED; @@ -235,7 +254,8 @@ unsigned long get_fb_unmapped_area(struct file *filp, unsigned long orig_addr, u align_goal = (64UL * 1024); do { - addr = get_area(NULL, orig_addr, len + (align_goal - PAGE_SIZE), pgoff, flags); + addr = mm_get_unmapped_area(NULL, orig_addr, + len + (align_goal - PAGE_SIZE), pgoff, flags); if (!(addr & ~PAGE_MASK)) { addr = (addr + (align_goal - 1UL)) & ~(align_goal - 1UL); break; @@ -253,7 +273,7 @@ unsigned long get_fb_unmapped_area(struct file *filp, unsigned long orig_addr, u * be obtained. */ if (addr & ~PAGE_MASK) - addr = get_area(NULL, orig_addr, len, pgoff, flags); + addr = mm_get_unmapped_area(NULL, orig_addr, len, pgoff, flags); return addr; } @@ -265,7 +285,7 @@ static unsigned long mmap_rnd(void) unsigned long rnd = 0UL; if (current->flags & PF_RANDOMIZE) { - unsigned long val = get_random_int(); + unsigned long val = get_random_long(); if (test_thread_flag(TIF_32BIT)) rnd = (val % (1UL << (23UL-PAGE_SHIFT))); else @@ -274,7 +294,7 @@ static unsigned long mmap_rnd(void) return rnd << PAGE_SHIFT; } -void arch_pick_mmap_layout(struct mm_struct *mm) +void arch_pick_mmap_layout(struct mm_struct *mm, const struct rlimit *rlim_stack) { unsigned long random_factor = mmap_rnd(); unsigned long gap; @@ -283,13 +303,13 @@ void arch_pick_mmap_layout(struct mm_struct *mm) * Fall back to the standard layout if the personality * bit is set, or if the expected stack growth is unlimited: */ - gap = rlimit(RLIMIT_STACK); + gap = rlim_stack->rlim_cur; if (!test_thread_flag(TIF_32BIT) || (current->personality & ADDR_COMPAT_LAYOUT) || gap == RLIM_INFINITY || sysctl_legacy_va_layout) { mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; - mm->get_unmapped_area = arch_get_unmapped_area; + mm_flags_clear(MMF_TOPDOWN, mm); } else { /* We know it's 32-bit */ unsigned long task_size = STACK_TOP32; @@ -300,7 +320,7 @@ void arch_pick_mmap_layout(struct mm_struct *mm) gap = (task_size / 6 * 5); mm->mmap_base = PAGE_ALIGN(task_size - gap - random_factor); - mm->get_unmapped_area = arch_get_unmapped_area_topdown; + mm_flags_set(MMF_TOPDOWN, mm); } } @@ -308,7 +328,7 @@ void arch_pick_mmap_layout(struct mm_struct *mm) * sys_pipe() is the normal C calling standard for creating * a pipe. It's not the way unix traditionally does this, though. */ -SYSCALL_DEFINE1(sparc_pipe_real, struct pt_regs *, regs) +SYSCALL_DEFINE0(sparc_pipe) { int fd[2]; int error; @@ -316,7 +336,7 @@ SYSCALL_DEFINE1(sparc_pipe_real, struct pt_regs *, regs) error = do_pipe_flags(fd, 0); if (error) goto out; - regs->u_regs[UREG_I1] = fd[1]; + current_pt_regs()->u_regs[UREG_I1] = fd[1]; error = fd[0]; out: return error; @@ -333,25 +353,28 @@ SYSCALL_DEFINE6(sparc_ipc, unsigned int, call, int, first, unsigned long, second { long err; + if (!IS_ENABLED(CONFIG_SYSVIPC)) + return -ENOSYS; + /* No need for backward compatibility. We can start fresh... */ - if (call <= SEMCTL) { + if (call <= SEMTIMEDOP) { switch (call) { case SEMOP: - err = sys_semtimedop(first, ptr, - (unsigned)second, NULL); + err = ksys_semtimedop(first, ptr, + (unsigned int)second, NULL); goto out; case SEMTIMEDOP: - err = sys_semtimedop(first, ptr, (unsigned)second, - (const struct timespec __user *) - (unsigned long) fifth); + err = ksys_semtimedop(first, ptr, (unsigned int)second, + (const struct __kernel_timespec __user *) + (unsigned long) fifth); goto out; case SEMGET: - err = sys_semget(first, (int)second, (int)third); + err = ksys_semget(first, (int)second, (int)third); goto out; case SEMCTL: { - err = sys_semctl(first, second, - (int)third | IPC_64, - (unsigned long) ptr); + err = ksys_old_semctl(first, second, + (int)third | IPC_64, + (unsigned long) ptr); goto out; } default: @@ -362,18 +385,18 @@ SYSCALL_DEFINE6(sparc_ipc, unsigned int, call, int, first, unsigned long, second if (call <= MSGCTL) { switch (call) { case MSGSND: - err = sys_msgsnd(first, ptr, (size_t)second, + err = ksys_msgsnd(first, ptr, (size_t)second, (int)third); goto out; case MSGRCV: - err = sys_msgrcv(first, ptr, (size_t)second, fifth, + err = ksys_msgrcv(first, ptr, (size_t)second, fifth, (int)third); goto out; case MSGGET: - err = sys_msgget((key_t)first, (int)second); + err = ksys_msgget((key_t)first, (int)second); goto out; case MSGCTL: - err = sys_msgctl(first, (int)second | IPC_64, ptr); + err = ksys_old_msgctl(first, (int)second | IPC_64, ptr); goto out; default: err = -ENOSYS; @@ -393,13 +416,13 @@ SYSCALL_DEFINE6(sparc_ipc, unsigned int, call, int, first, unsigned long, second goto out; } case SHMDT: - err = sys_shmdt(ptr); + err = ksys_shmdt(ptr); goto out; case SHMGET: - err = sys_shmget(first, (size_t)second, (int)third); + err = ksys_shmget(first, (size_t)second, (int)third); goto out; case SHMCTL: - err = sys_shmctl(first, (int)second | IPC_64, ptr); + err = ksys_old_shmctl(first, (int)second | IPC_64, ptr); goto out; default: err = -ENOSYS; @@ -414,7 +437,7 @@ out: SYSCALL_DEFINE1(sparc64_personality, unsigned long, personality) { - int ret; + long ret; if (personality(current->personality) == PER_LINUX32 && personality(personality) == PER_LINUX) @@ -456,7 +479,7 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, goto out; if (off & ~PAGE_MASK) goto out; - retval = sys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); + retval = ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); out: return retval; } @@ -478,10 +501,10 @@ SYSCALL_DEFINE5(64_mremap, unsigned long, addr, unsigned long, old_len, return sys_mremap(addr, old_len, new_len, flags, new_addr); } -/* we come to here via sys_nis_syscall so it can setup the regs argument */ -asmlinkage unsigned long c_sys_nis_syscall(struct pt_regs *regs) +SYSCALL_DEFINE0(nis_syscall) { static int count; + struct pt_regs *regs = current_pt_regs(); /* Don't make the system unusable, if someone goes stuck */ if (count++ > 5) @@ -499,7 +522,7 @@ asmlinkage unsigned long c_sys_nis_syscall(struct pt_regs *regs) asmlinkage void sparc_breakpoint(struct pt_regs *regs) { - siginfo_t info; + enum ctx_state prev_state = exception_enter(); if (test_thread_flag(TIF_32BIT)) { regs->tpc &= 0xffffffff; @@ -508,42 +531,97 @@ asmlinkage void sparc_breakpoint(struct pt_regs *regs) #ifdef DEBUG_SPARC_BREAKPOINT printk ("TRAP: Entering kernel PC=%lx, nPC=%lx\n", regs->tpc, regs->tnpc); #endif - info.si_signo = SIGTRAP; - info.si_errno = 0; - info.si_code = TRAP_BRKPT; - info.si_addr = (void __user *)regs->tpc; - info.si_trapno = 0; - force_sig_info(SIGTRAP, &info, current); + force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->tpc); #ifdef DEBUG_SPARC_BREAKPOINT printk ("TRAP: Returning to space: PC=%lx nPC=%lx\n", regs->tpc, regs->tnpc); #endif + exception_exit(prev_state); } -extern void check_pending(int signum); - SYSCALL_DEFINE2(getdomainname, char __user *, name, int, len) { - int nlen, err; + int nlen, err; + char tmp[__NEW_UTS_LEN + 1]; if (len < 0) return -EINVAL; - down_read(&uts_sem); - + down_read(&uts_sem); + nlen = strlen(utsname()->domainname) + 1; err = -EINVAL; if (nlen > len) - goto out; + goto out_unlock; + memcpy(tmp, utsname()->domainname, nlen); - err = -EFAULT; - if (!copy_to_user(name, utsname()->domainname, nlen)) - err = 0; + up_read(&uts_sem); -out: + if (copy_to_user(name, tmp, nlen)) + return -EFAULT; + return 0; + +out_unlock: up_read(&uts_sem); return err; } +SYSCALL_DEFINE1(sparc_adjtimex, struct __kernel_timex __user *, txc_p) +{ + struct __kernel_timex txc; + struct __kernel_old_timeval *tv = (void *)&txc.time; + int ret; + + /* Copy the user data space into the kernel copy + * structure. But bear in mind that the structures + * may change + */ + if (copy_from_user(&txc, txc_p, sizeof(txc))) + return -EFAULT; + + /* + * override for sparc64 specific timeval type: tv_usec + * is 32 bit wide instead of 64-bit in __kernel_timex + */ + txc.time.tv_usec = tv->tv_usec; + ret = do_adjtimex(&txc); + tv->tv_usec = txc.time.tv_usec; + + return copy_to_user(txc_p, &txc, sizeof(txc)) ? -EFAULT : ret; +} + +SYSCALL_DEFINE2(sparc_clock_adjtime, const clockid_t, which_clock, + struct __kernel_timex __user *, txc_p) +{ + struct __kernel_timex txc; + struct __kernel_old_timeval *tv = (void *)&txc.time; + int ret; + + if (!IS_ENABLED(CONFIG_POSIX_TIMERS)) { + pr_err_once("process %d (%s) attempted a POSIX timer syscall " + "while CONFIG_POSIX_TIMERS is not set\n", + current->pid, current->comm); + + return -ENOSYS; + } + + /* Copy the user data space into the kernel copy + * structure. But bear in mind that the structures + * may change + */ + if (copy_from_user(&txc, txc_p, sizeof(txc))) + return -EFAULT; + + /* + * override for sparc64 specific timeval type: tv_usec + * is 32 bit wide instead of 64-bit in __kernel_timex + */ + txc.time.tv_usec = tv->tv_usec; + ret = do_clock_adjtime(which_clock, &txc); + tv->tv_usec = txc.time.tv_usec; + + return copy_to_user(txc_p, &txc, sizeof(txc)) ? -EFAULT : ret; +} + SYSCALL_DEFINE5(utrap_install, utrap_entry_t, type, utrap_handler_t, new_p, utrap_handler_t, new_d, utrap_handler_t __user *, old_p, @@ -569,7 +647,8 @@ SYSCALL_DEFINE5(utrap_install, utrap_entry_t, type, } if (!current_thread_info()->utraps) { current_thread_info()->utraps = - kzalloc((UT_TRAP_INSTRUCTION_31+1)*sizeof(long), GFP_KERNEL); + kcalloc(UT_TRAP_INSTRUCTION_31 + 1, sizeof(long), + GFP_KERNEL); if (!current_thread_info()->utraps) return -ENOMEM; current_thread_info()->utraps[0] = 1; @@ -579,8 +658,9 @@ SYSCALL_DEFINE5(utrap_install, utrap_entry_t, type, unsigned long *p = current_thread_info()->utraps; current_thread_info()->utraps = - kmalloc((UT_TRAP_INSTRUCTION_31+1)*sizeof(long), - GFP_KERNEL); + kmalloc_array(UT_TRAP_INSTRUCTION_31 + 1, + sizeof(long), + GFP_KERNEL); if (!current_thread_info()->utraps) { current_thread_info()->utraps = p; return -ENOMEM; @@ -604,9 +684,9 @@ SYSCALL_DEFINE5(utrap_install, utrap_entry_t, type, return 0; } -asmlinkage long sparc_memory_ordering(unsigned long model, - struct pt_regs *regs) +SYSCALL_DEFINE1(memory_ordering, unsigned long, model) { + struct pt_regs *regs = current_pt_regs(); if (model >= 3) return -EINVAL; regs->tstate = (regs->tstate & ~TSTATE_MM) | (model << 14); @@ -640,7 +720,7 @@ SYSCALL_DEFINE5(rt_sigaction, int, sig, const struct sigaction __user *, act, return ret; } -asmlinkage long sys_kern_features(void) +SYSCALL_DEFINE0(kern_features) { return KERN_FEATURE_MIXED_MODE_STACK; } diff --git a/arch/sparc/kernel/syscalls.S b/arch/sparc/kernel/syscalls.S index 22a1098961f5..0e8ab0602c36 100644 --- a/arch/sparc/kernel/syscalls.S +++ b/arch/sparc/kernel/syscalls.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* SunOS's execv() call only specifies the argv argument, the * environment settings are the same as the calling processes. */ @@ -6,6 +7,11 @@ sys64_execve: jmpl %g1, %g0 flushw +sys64_execveat: + set sys_execveat, %g1 + jmpl %g1, %g0 + flushw + #ifdef CONFIG_COMPAT sunos_execv: mov %g0, %o2 @@ -13,18 +19,14 @@ sys32_execve: set compat_sys_execve, %g1 jmpl %g1, %g0 flushw + +sys32_execveat: + set compat_sys_execveat, %g1 + jmpl %g1, %g0 + flushw #endif .align 32 -sys_sparc_pipe: - ba,pt %xcc, sys_sparc_pipe_real - add %sp, PTREGS_OFF, %o0 -sys_nis_syscall: - ba,pt %xcc, c_sys_nis_syscall - add %sp, PTREGS_OFF, %o0 -sys_memory_ordering: - ba,pt %xcc, sparc_memory_ordering - add %sp, PTREGS_OFF, %o1 #ifdef CONFIG_COMPAT sys32_sigstack: ba,pt %xcc, do_sys32_sigstack @@ -52,7 +54,7 @@ sys32_rt_sigreturn: #endif .align 32 1: ldx [%g6 + TI_FLAGS], %l5 - andcc %l5, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 + andcc %l5, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 be,pt %icc, rtrap nop call syscall_trace_leave @@ -84,22 +86,25 @@ sys32_rt_sigreturn: * during system calls... */ .align 32 -sys_vfork: /* Under Linux, vfork and fork are just special cases of clone. */ - sethi %hi(0x4000 | 0x0100 | SIGCHLD), %o0 - or %o0, %lo(0x4000 | 0x0100 | SIGCHLD), %o0 - ba,pt %xcc, sys_clone +sys_vfork: + flushw + ba,pt %xcc, sparc_vfork + add %sp, PTREGS_OFF, %o0 + + .align 32 sys_fork: - clr %o1 - mov SIGCHLD, %o0 + flushw + ba,pt %xcc, sparc_fork + add %sp, PTREGS_OFF, %o0 + + .align 32 sys_clone: flushw - movrz %o1, %fp, %o1 - mov 0, %o3 - ba,pt %xcc, sparc_do_fork - add %sp, PTREGS_OFF, %o2 + ba,pt %xcc, sparc_clone + add %sp, PTREGS_OFF, %o0 - .globl ret_from_syscall -ret_from_syscall: + .globl ret_from_fork +ret_from_fork: /* Clear current_thread_info()->new_child. */ stb %g0, [%g6 + TI_NEW_CHILD] call schedule_tail @@ -148,11 +153,29 @@ linux_syscall_trace32: add %sp, PTREGS_OFF, %o0 brnz,pn %o0, 3f mov -ENOSYS, %o0 + + /* Syscall tracing can modify the registers. */ + ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 + sethi %hi(sys_call_table32), %l7 + ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0 + or %l7, %lo(sys_call_table32), %l7 + ldx [%sp + PTREGS_OFF + PT_V9_I1], %i1 + ldx [%sp + PTREGS_OFF + PT_V9_I2], %i2 + ldx [%sp + PTREGS_OFF + PT_V9_I3], %i3 + ldx [%sp + PTREGS_OFF + PT_V9_I4], %i4 + ldx [%sp + PTREGS_OFF + PT_V9_I5], %i5 + + cmp %g1, NR_syscalls + bgeu,pn %xcc, 3f + mov -ENOSYS, %o0 + + sll %g1, 2, %l4 srl %i0, 0, %o0 + lduw [%l7 + %l4], %l7 srl %i4, 0, %o4 srl %i1, 0, %o1 srl %i2, 0, %o2 - ba,pt %xcc, 2f + ba,pt %xcc, 5f srl %i3, 0, %o3 linux_syscall_trace: @@ -160,7 +183,25 @@ linux_syscall_trace: add %sp, PTREGS_OFF, %o0 brnz,pn %o0, 3f mov -ENOSYS, %o0 + + /* Syscall tracing can modify the registers. */ + ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 + sethi %hi(sys_call_table64), %l7 + ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0 + or %l7, %lo(sys_call_table64), %l7 + ldx [%sp + PTREGS_OFF + PT_V9_I1], %i1 + ldx [%sp + PTREGS_OFF + PT_V9_I2], %i2 + ldx [%sp + PTREGS_OFF + PT_V9_I3], %i3 + ldx [%sp + PTREGS_OFF + PT_V9_I4], %i4 + ldx [%sp + PTREGS_OFF + PT_V9_I5], %i5 + + cmp %g1, NR_syscalls + bgeu,pn %xcc, 3f + mov -ENOSYS, %o0 + + sll %g1, 2, %l4 mov %i0, %o0 + lduw [%l7 + %l4], %l7 mov %i1, %o1 mov %i2, %o2 mov %i3, %o3 @@ -182,14 +223,15 @@ linux_sparc_syscall32: srl %i1, 0, %o1 ! IEU0 Group ldx [%g6 + TI_FLAGS], %l0 ! Load - srl %i5, 0, %o5 ! IEU1 + srl %i3, 0, %o3 ! IEU0 srl %i2, 0, %o2 ! IEU0 Group - andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 + andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 bne,pn %icc, linux_syscall_trace32 ! CTI mov %i0, %l5 ! IEU1 - call %l7 ! CTI Group brk forced - srl %i3, 0, %o3 ! IEU0 - ba,a,pt %xcc, 3f +5: call %l7 ! CTI Group brk forced + srl %i5, 0, %o5 ! IEU1 + ba,pt %xcc, 3f + sra %o0, 0, %o0 /* Linux native system calls enter here... */ .align 32 @@ -207,7 +249,7 @@ linux_sparc_syscall: mov %i3, %o3 ! IEU1 mov %i4, %o4 ! IEU0 Group - andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 + andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 bne,pn %icc, linux_syscall_trace ! CTI Group mov %i0, %l5 ! IEU0 2: call %l7 ! CTI Group brk forced @@ -217,13 +259,12 @@ linux_sparc_syscall: 3: stx %o0, [%sp + PTREGS_OFF + PT_V9_I0] ret_sys_call: ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %g3 - sra %o0, 0, %o0 mov %ulo(TSTATE_XCARRY | TSTATE_ICARRY), %g2 sllx %g2, 32, %g2 cmp %o0, -ERESTART_RESTARTBLOCK bgeu,pn %xcc, 1f - andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 + andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc 2: diff --git a/arch/sparc/kernel/syscalls/Makefile b/arch/sparc/kernel/syscalls/Makefile new file mode 100644 index 000000000000..8440c16dfb22 --- /dev/null +++ b/arch/sparc/kernel/syscalls/Makefile @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: GPL-2.0 +kapi := arch/$(SRCARCH)/include/generated/asm +uapi := arch/$(SRCARCH)/include/generated/uapi/asm + +$(shell mkdir -p $(uapi) $(kapi)) + +syscall := $(src)/syscall.tbl +syshdr := $(srctree)/scripts/syscallhdr.sh +systbl := $(srctree)/scripts/syscalltbl.sh + +quiet_cmd_syshdr = SYSHDR $@ + cmd_syshdr = $(CONFIG_SHELL) $(syshdr) --emit-nr --abis common,$* $< $@ + +quiet_cmd_systbl = SYSTBL $@ + cmd_systbl = $(CONFIG_SHELL) $(systbl) --abis common,$* $< $@ + +$(uapi)/unistd_%.h: $(syscall) $(syshdr) FORCE + $(call if_changed,syshdr) + +$(kapi)/syscall_table_%.h: $(syscall) $(systbl) FORCE + $(call if_changed,systbl) + +uapisyshdr-y += unistd_32.h unistd_64.h +kapisyshdr-y += syscall_table_32.h \ + syscall_table_64.h + +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) + +PHONY += all +all: $(uapisyshdr-y) $(kapisyshdr-y) + @: diff --git a/arch/sparc/kernel/syscalls/syscall.tbl b/arch/sparc/kernel/syscalls/syscall.tbl new file mode 100644 index 000000000000..39aa26b6a50b --- /dev/null +++ b/arch/sparc/kernel/syscalls/syscall.tbl @@ -0,0 +1,518 @@ +# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +# +# system call numbers and entry vectors for sparc +# +# The format is: +# <number> <abi> <name> <entry point> <compat entry point> +# +# The <abi> can be common, 64, or 32 for this file. +# +0 common restart_syscall sys_restart_syscall +1 32 exit sys_exit sparc_exit +1 64 exit sparc_exit +2 common fork sys_fork +3 common read sys_read +4 common write sys_write +5 common open sys_open compat_sys_open +6 common close sys_close +7 common wait4 sys_wait4 compat_sys_wait4 +8 common creat sys_creat +9 common link sys_link +10 common unlink sys_unlink +11 32 execv sunos_execv +11 64 execv sys_nis_syscall +12 common chdir sys_chdir +13 32 chown sys_chown16 +13 64 chown sys_chown +14 common mknod sys_mknod +15 common chmod sys_chmod +16 32 lchown sys_lchown16 +16 64 lchown sys_lchown +17 common brk sys_brk +18 common perfctr sys_nis_syscall +19 common lseek sys_lseek compat_sys_lseek +20 common getpid sys_getpid +21 common capget sys_capget +22 common capset sys_capset +23 32 setuid sys_setuid16 +23 64 setuid sys_setuid +24 32 getuid sys_getuid16 +24 64 getuid sys_getuid +25 common vmsplice sys_vmsplice +26 common ptrace sys_ptrace compat_sys_ptrace +27 common alarm sys_alarm +28 common sigaltstack sys_sigaltstack compat_sys_sigaltstack +29 32 pause sys_pause +29 64 pause sys_nis_syscall +30 32 utime sys_utime32 +30 64 utime sys_utime +31 32 lchown32 sys_lchown +32 32 fchown32 sys_fchown +33 common access sys_access +34 common nice sys_nice +35 32 chown32 sys_chown +36 common sync sys_sync +37 common kill sys_kill +38 common stat sys_newstat compat_sys_newstat +39 32 sendfile sys_sendfile compat_sys_sendfile +39 64 sendfile sys_sendfile64 +40 common lstat sys_newlstat compat_sys_newlstat +41 common dup sys_dup +42 common pipe sys_sparc_pipe +43 common times sys_times compat_sys_times +44 32 getuid32 sys_getuid +45 common umount2 sys_umount +46 32 setgid sys_setgid16 +46 64 setgid sys_setgid +47 32 getgid sys_getgid16 +47 64 getgid sys_getgid +48 common signal sys_signal +49 32 geteuid sys_geteuid16 +49 64 geteuid sys_geteuid +50 32 getegid sys_getegid16 +50 64 getegid sys_getegid +51 common acct sys_acct +52 64 memory_ordering sys_memory_ordering +53 32 getgid32 sys_getgid +54 common ioctl sys_ioctl compat_sys_ioctl +55 common reboot sys_reboot +56 32 mmap2 sys_mmap2 sys32_mmap2 +57 common symlink sys_symlink +58 common readlink sys_readlink +59 32 execve sys_execve sys32_execve +59 64 execve sys64_execve +60 common umask sys_umask +61 common chroot sys_chroot +62 common fstat sys_newfstat compat_sys_newfstat +63 common fstat64 sys_fstat64 compat_sys_fstat64 +64 common getpagesize sys_getpagesize +65 common msync sys_msync +66 common vfork sys_vfork +67 common pread64 sys_pread64 compat_sys_pread64 +68 common pwrite64 sys_pwrite64 compat_sys_pwrite64 +69 32 geteuid32 sys_geteuid +70 32 getegid32 sys_getegid +71 common mmap sys_mmap +72 32 setreuid32 sys_setreuid +73 32 munmap sys_munmap +73 64 munmap sys_64_munmap +74 common mprotect sys_mprotect +75 common madvise sys_madvise +76 common vhangup sys_vhangup +77 32 truncate64 sys_truncate64 compat_sys_truncate64 +78 common mincore sys_mincore +79 32 getgroups sys_getgroups16 +79 64 getgroups sys_getgroups +80 32 setgroups sys_setgroups16 +80 64 setgroups sys_setgroups +81 common getpgrp sys_getpgrp +82 32 setgroups32 sys_setgroups +83 common setitimer sys_setitimer compat_sys_setitimer +84 32 ftruncate64 sys_ftruncate64 compat_sys_ftruncate64 +85 common swapon sys_swapon +86 common getitimer sys_getitimer compat_sys_getitimer +87 32 setuid32 sys_setuid +88 common sethostname sys_sethostname +89 32 setgid32 sys_setgid +90 common dup2 sys_dup2 +91 32 setfsuid32 sys_setfsuid +92 common fcntl sys_fcntl compat_sys_fcntl +93 common select sys_select compat_sys_select +94 32 setfsgid32 sys_setfsgid +95 common fsync sys_fsync +96 common setpriority sys_setpriority +97 common socket sys_socket +98 common connect sys_connect +99 common accept sys_accept +100 common getpriority sys_getpriority +101 common rt_sigreturn sys_rt_sigreturn sys32_rt_sigreturn +102 common rt_sigaction sys_rt_sigaction compat_sys_rt_sigaction +103 common rt_sigprocmask sys_rt_sigprocmask compat_sys_rt_sigprocmask +104 common rt_sigpending sys_rt_sigpending compat_sys_rt_sigpending +105 32 rt_sigtimedwait sys_rt_sigtimedwait_time32 compat_sys_rt_sigtimedwait_time32 +105 64 rt_sigtimedwait sys_rt_sigtimedwait +106 common rt_sigqueueinfo sys_rt_sigqueueinfo compat_sys_rt_sigqueueinfo +107 common rt_sigsuspend sys_rt_sigsuspend compat_sys_rt_sigsuspend +108 32 setresuid32 sys_setresuid +108 64 setresuid sys_setresuid +109 32 getresuid32 sys_getresuid +109 64 getresuid sys_getresuid +110 32 setresgid32 sys_setresgid +110 64 setresgid sys_setresgid +111 32 getresgid32 sys_getresgid +111 64 getresgid sys_getresgid +112 32 setregid32 sys_setregid +113 common recvmsg sys_recvmsg compat_sys_recvmsg +114 common sendmsg sys_sendmsg compat_sys_sendmsg +115 32 getgroups32 sys_getgroups +116 common gettimeofday sys_gettimeofday compat_sys_gettimeofday +117 common getrusage sys_getrusage compat_sys_getrusage +118 common getsockopt sys_getsockopt sys_getsockopt +119 common getcwd sys_getcwd +120 common readv sys_readv +121 common writev sys_writev +122 common settimeofday sys_settimeofday compat_sys_settimeofday +123 32 fchown sys_fchown16 +123 64 fchown sys_fchown +124 common fchmod sys_fchmod +125 common recvfrom sys_recvfrom compat_sys_recvfrom +126 32 setreuid sys_setreuid16 +126 64 setreuid sys_setreuid +127 32 setregid sys_setregid16 +127 64 setregid sys_setregid +128 common rename sys_rename +129 common truncate sys_truncate compat_sys_truncate +130 common ftruncate sys_ftruncate compat_sys_ftruncate +131 common flock sys_flock +132 common lstat64 sys_lstat64 compat_sys_lstat64 +133 common sendto sys_sendto +134 common shutdown sys_shutdown +135 common socketpair sys_socketpair +136 common mkdir sys_mkdir +137 common rmdir sys_rmdir +138 32 utimes sys_utimes_time32 +138 64 utimes sys_utimes +139 common stat64 sys_stat64 compat_sys_stat64 +140 common sendfile64 sys_sendfile64 +141 common getpeername sys_getpeername +142 32 futex sys_futex_time32 +142 64 futex sys_futex +143 common gettid sys_gettid +144 common getrlimit sys_getrlimit compat_sys_getrlimit +145 common setrlimit sys_setrlimit compat_sys_setrlimit +146 common pivot_root sys_pivot_root +147 common prctl sys_prctl +148 common pciconfig_read sys_pciconfig_read +149 common pciconfig_write sys_pciconfig_write +150 common getsockname sys_getsockname +151 common inotify_init sys_inotify_init +152 common inotify_add_watch sys_inotify_add_watch +153 common poll sys_poll +154 common getdents64 sys_getdents64 +155 32 fcntl64 sys_fcntl64 compat_sys_fcntl64 +156 common inotify_rm_watch sys_inotify_rm_watch +157 common statfs sys_statfs compat_sys_statfs +158 common fstatfs sys_fstatfs compat_sys_fstatfs +159 common umount sys_oldumount +160 common sched_set_affinity sys_sched_setaffinity compat_sys_sched_setaffinity +161 common sched_get_affinity sys_sched_getaffinity compat_sys_sched_getaffinity +162 common getdomainname sys_getdomainname +163 common setdomainname sys_setdomainname +164 64 utrap_install sys_utrap_install +165 common quotactl sys_quotactl +166 common set_tid_address sys_set_tid_address +167 common mount sys_mount +168 common ustat sys_ustat compat_sys_ustat +169 common setxattr sys_setxattr +170 common lsetxattr sys_lsetxattr +171 common fsetxattr sys_fsetxattr +172 common getxattr sys_getxattr +173 common lgetxattr sys_lgetxattr +174 common getdents sys_getdents compat_sys_getdents +175 common setsid sys_setsid +176 common fchdir sys_fchdir +177 common fgetxattr sys_fgetxattr +178 common listxattr sys_listxattr +179 common llistxattr sys_llistxattr +180 common flistxattr sys_flistxattr +181 common removexattr sys_removexattr +182 common lremovexattr sys_lremovexattr +183 32 sigpending sys_sigpending compat_sys_sigpending +183 64 sigpending sys_nis_syscall +184 common query_module sys_ni_syscall +185 common setpgid sys_setpgid +186 common fremovexattr sys_fremovexattr +187 common tkill sys_tkill +188 32 exit_group sys_exit_group sparc_exit_group +188 64 exit_group sparc_exit_group +189 common uname sys_newuname +190 common init_module sys_init_module +191 32 personality sys_personality sys_sparc64_personality +191 64 personality sys_sparc64_personality +192 32 remap_file_pages sys_sparc_remap_file_pages sys_remap_file_pages +192 64 remap_file_pages sys_remap_file_pages +193 common epoll_create sys_epoll_create +194 common epoll_ctl sys_epoll_ctl +195 common epoll_wait sys_epoll_wait +196 common ioprio_set sys_ioprio_set +197 common getppid sys_getppid +198 32 sigaction sys_sparc_sigaction compat_sys_sparc_sigaction +198 64 sigaction sys_nis_syscall +199 common sgetmask sys_sgetmask +200 common ssetmask sys_ssetmask +201 32 sigsuspend sys_sigsuspend +201 64 sigsuspend sys_nis_syscall +202 common oldlstat sys_newlstat compat_sys_newlstat +203 common uselib sys_uselib +204 32 readdir sys_old_readdir compat_sys_old_readdir +204 64 readdir sys_nis_syscall +205 common readahead sys_readahead compat_sys_readahead +206 common socketcall sys_socketcall compat_sys_socketcall +207 common syslog sys_syslog +208 common lookup_dcookie sys_ni_syscall +209 common fadvise64 sys_fadvise64 compat_sys_fadvise64 +210 common fadvise64_64 sys_fadvise64_64 compat_sys_fadvise64_64 +211 common tgkill sys_tgkill +212 common waitpid sys_waitpid +213 common swapoff sys_swapoff +214 common sysinfo sys_sysinfo compat_sys_sysinfo +215 32 ipc sys_ipc compat_sys_ipc +215 64 ipc sys_sparc_ipc +216 32 sigreturn sys_sigreturn sys32_sigreturn +216 64 sigreturn sys_nis_syscall +217 common clone sys_clone +218 common ioprio_get sys_ioprio_get +219 32 adjtimex sys_adjtimex_time32 +219 64 adjtimex sys_sparc_adjtimex +220 32 sigprocmask sys_sigprocmask compat_sys_sigprocmask +220 64 sigprocmask sys_nis_syscall +221 common create_module sys_ni_syscall +222 common delete_module sys_delete_module +223 common get_kernel_syms sys_ni_syscall +224 common getpgid sys_getpgid +225 common bdflush sys_ni_syscall +226 common sysfs sys_sysfs +227 common afs_syscall sys_nis_syscall +228 common setfsuid sys_setfsuid16 +229 common setfsgid sys_setfsgid16 +230 common _newselect sys_select compat_sys_select +231 32 time sys_time32 +232 common splice sys_splice +233 32 stime sys_stime32 +233 64 stime sys_stime +234 common statfs64 sys_statfs64 compat_sys_statfs64 +235 common fstatfs64 sys_fstatfs64 compat_sys_fstatfs64 +236 common _llseek sys_llseek +237 common mlock sys_mlock +238 common munlock sys_munlock +239 common mlockall sys_mlockall +240 common munlockall sys_munlockall +241 common sched_setparam sys_sched_setparam +242 common sched_getparam sys_sched_getparam +243 common sched_setscheduler sys_sched_setscheduler +244 common sched_getscheduler sys_sched_getscheduler +245 common sched_yield sys_sched_yield +246 common sched_get_priority_max sys_sched_get_priority_max +247 common sched_get_priority_min sys_sched_get_priority_min +248 32 sched_rr_get_interval sys_sched_rr_get_interval_time32 +248 64 sched_rr_get_interval sys_sched_rr_get_interval +249 32 nanosleep sys_nanosleep_time32 +249 64 nanosleep sys_nanosleep +250 32 mremap sys_mremap +250 64 mremap sys_64_mremap +251 common _sysctl sys_ni_syscall +252 common getsid sys_getsid +253 common fdatasync sys_fdatasync +254 32 nfsservctl sys_ni_syscall sys_nis_syscall +254 64 nfsservctl sys_nis_syscall +255 common sync_file_range sys_sync_file_range compat_sys_sync_file_range +256 32 clock_settime sys_clock_settime32 +256 64 clock_settime sys_clock_settime +257 32 clock_gettime sys_clock_gettime32 +257 64 clock_gettime sys_clock_gettime +258 32 clock_getres sys_clock_getres_time32 +258 64 clock_getres sys_clock_getres +259 32 clock_nanosleep sys_clock_nanosleep_time32 +259 64 clock_nanosleep sys_clock_nanosleep +260 common sched_getaffinity sys_sched_getaffinity compat_sys_sched_getaffinity +261 common sched_setaffinity sys_sched_setaffinity compat_sys_sched_setaffinity +262 32 timer_settime sys_timer_settime32 +262 64 timer_settime sys_timer_settime +263 32 timer_gettime sys_timer_gettime32 +263 64 timer_gettime sys_timer_gettime +264 common timer_getoverrun sys_timer_getoverrun +265 common timer_delete sys_timer_delete +266 common timer_create sys_timer_create compat_sys_timer_create +# 267 was vserver +267 common vserver sys_nis_syscall +268 common io_setup sys_io_setup compat_sys_io_setup +269 common io_destroy sys_io_destroy +270 common io_submit sys_io_submit compat_sys_io_submit +271 common io_cancel sys_io_cancel +272 32 io_getevents sys_io_getevents_time32 +272 64 io_getevents sys_io_getevents +273 common mq_open sys_mq_open compat_sys_mq_open +274 common mq_unlink sys_mq_unlink +275 32 mq_timedsend sys_mq_timedsend_time32 +275 64 mq_timedsend sys_mq_timedsend +276 32 mq_timedreceive sys_mq_timedreceive_time32 +276 64 mq_timedreceive sys_mq_timedreceive +277 common mq_notify sys_mq_notify compat_sys_mq_notify +278 common mq_getsetattr sys_mq_getsetattr compat_sys_mq_getsetattr +279 common waitid sys_waitid compat_sys_waitid +280 common tee sys_tee +281 common add_key sys_add_key +282 common request_key sys_request_key +283 common keyctl sys_keyctl compat_sys_keyctl +284 common openat sys_openat compat_sys_openat +285 common mkdirat sys_mkdirat +286 common mknodat sys_mknodat +287 common fchownat sys_fchownat +288 32 futimesat sys_futimesat_time32 +288 64 futimesat sys_futimesat +289 common fstatat64 sys_fstatat64 compat_sys_fstatat64 +290 common unlinkat sys_unlinkat +291 common renameat sys_renameat +292 common linkat sys_linkat +293 common symlinkat sys_symlinkat +294 common readlinkat sys_readlinkat +295 common fchmodat sys_fchmodat +296 common faccessat sys_faccessat +297 32 pselect6 sys_pselect6_time32 compat_sys_pselect6_time32 +297 64 pselect6 sys_pselect6 +298 32 ppoll sys_ppoll_time32 compat_sys_ppoll_time32 +298 64 ppoll sys_ppoll +299 common unshare sys_unshare +300 common set_robust_list sys_set_robust_list compat_sys_set_robust_list +301 common get_robust_list sys_get_robust_list compat_sys_get_robust_list +302 common migrate_pages sys_migrate_pages +303 common mbind sys_mbind +304 common get_mempolicy sys_get_mempolicy +305 common set_mempolicy sys_set_mempolicy +306 common kexec_load sys_kexec_load compat_sys_kexec_load +307 common move_pages sys_move_pages +308 common getcpu sys_getcpu +309 common epoll_pwait sys_epoll_pwait compat_sys_epoll_pwait +310 32 utimensat sys_utimensat_time32 +310 64 utimensat sys_utimensat +311 common signalfd sys_signalfd compat_sys_signalfd +312 common timerfd_create sys_timerfd_create +313 common eventfd sys_eventfd +314 common fallocate sys_fallocate compat_sys_fallocate +315 32 timerfd_settime sys_timerfd_settime32 +315 64 timerfd_settime sys_timerfd_settime +316 32 timerfd_gettime sys_timerfd_gettime32 +316 64 timerfd_gettime sys_timerfd_gettime +317 common signalfd4 sys_signalfd4 compat_sys_signalfd4 +318 common eventfd2 sys_eventfd2 +319 common epoll_create1 sys_epoll_create1 +320 common dup3 sys_dup3 +321 common pipe2 sys_pipe2 +322 common inotify_init1 sys_inotify_init1 +323 common accept4 sys_accept4 +324 common preadv sys_preadv compat_sys_preadv +325 common pwritev sys_pwritev compat_sys_pwritev +326 common rt_tgsigqueueinfo sys_rt_tgsigqueueinfo compat_sys_rt_tgsigqueueinfo +327 common perf_event_open sys_perf_event_open +328 32 recvmmsg sys_recvmmsg_time32 compat_sys_recvmmsg_time32 +328 64 recvmmsg sys_recvmmsg +329 common fanotify_init sys_fanotify_init +330 common fanotify_mark sys_fanotify_mark compat_sys_fanotify_mark +331 common prlimit64 sys_prlimit64 +332 common name_to_handle_at sys_name_to_handle_at +333 common open_by_handle_at sys_open_by_handle_at compat_sys_open_by_handle_at +334 32 clock_adjtime sys_clock_adjtime32 +334 64 clock_adjtime sys_sparc_clock_adjtime +335 common syncfs sys_syncfs +336 common sendmmsg sys_sendmmsg compat_sys_sendmmsg +337 common setns sys_setns +338 common process_vm_readv sys_process_vm_readv +339 common process_vm_writev sys_process_vm_writev +340 32 kern_features sys_ni_syscall sys_kern_features +340 64 kern_features sys_kern_features +341 common kcmp sys_kcmp +342 common finit_module sys_finit_module +343 common sched_setattr sys_sched_setattr +344 common sched_getattr sys_sched_getattr +345 common renameat2 sys_renameat2 +346 common seccomp sys_seccomp +347 common getrandom sys_getrandom +348 common memfd_create sys_memfd_create +349 common bpf sys_bpf +350 32 execveat sys_execveat sys32_execveat +350 64 execveat sys64_execveat +351 common membarrier sys_membarrier +352 common userfaultfd sys_userfaultfd +353 common bind sys_bind +354 common listen sys_listen +355 common setsockopt sys_setsockopt sys_setsockopt +356 common mlock2 sys_mlock2 +357 common copy_file_range sys_copy_file_range +358 common preadv2 sys_preadv2 compat_sys_preadv2 +359 common pwritev2 sys_pwritev2 compat_sys_pwritev2 +360 common statx sys_statx +361 32 io_pgetevents sys_io_pgetevents_time32 compat_sys_io_pgetevents +361 64 io_pgetevents sys_io_pgetevents +362 common pkey_mprotect sys_pkey_mprotect +363 common pkey_alloc sys_pkey_alloc +364 common pkey_free sys_pkey_free +365 common rseq sys_rseq +# room for arch specific syscalls +392 64 semtimedop sys_semtimedop +393 common semget sys_semget +394 common semctl sys_semctl compat_sys_semctl +395 common shmget sys_shmget +396 common shmctl sys_shmctl compat_sys_shmctl +397 common shmat sys_shmat compat_sys_shmat +398 common shmdt sys_shmdt +399 common msgget sys_msgget +400 common msgsnd sys_msgsnd compat_sys_msgsnd +401 common msgrcv sys_msgrcv compat_sys_msgrcv +402 common msgctl sys_msgctl compat_sys_msgctl +403 32 clock_gettime64 sys_clock_gettime sys_clock_gettime +404 32 clock_settime64 sys_clock_settime sys_clock_settime +405 32 clock_adjtime64 sys_clock_adjtime sys_clock_adjtime +406 32 clock_getres_time64 sys_clock_getres sys_clock_getres +407 32 clock_nanosleep_time64 sys_clock_nanosleep sys_clock_nanosleep +408 32 timer_gettime64 sys_timer_gettime sys_timer_gettime +409 32 timer_settime64 sys_timer_settime sys_timer_settime +410 32 timerfd_gettime64 sys_timerfd_gettime sys_timerfd_gettime +411 32 timerfd_settime64 sys_timerfd_settime sys_timerfd_settime +412 32 utimensat_time64 sys_utimensat sys_utimensat +413 32 pselect6_time64 sys_pselect6 compat_sys_pselect6_time64 +414 32 ppoll_time64 sys_ppoll compat_sys_ppoll_time64 +416 32 io_pgetevents_time64 sys_io_pgetevents compat_sys_io_pgetevents_time64 +417 32 recvmmsg_time64 sys_recvmmsg compat_sys_recvmmsg_time64 +418 32 mq_timedsend_time64 sys_mq_timedsend sys_mq_timedsend +419 32 mq_timedreceive_time64 sys_mq_timedreceive sys_mq_timedreceive +420 32 semtimedop_time64 sys_semtimedop sys_semtimedop +421 32 rt_sigtimedwait_time64 sys_rt_sigtimedwait compat_sys_rt_sigtimedwait_time64 +422 32 futex_time64 sys_futex sys_futex +423 32 sched_rr_get_interval_time64 sys_sched_rr_get_interval sys_sched_rr_get_interval +424 common pidfd_send_signal sys_pidfd_send_signal +425 common io_uring_setup sys_io_uring_setup +426 common io_uring_enter sys_io_uring_enter +427 common io_uring_register sys_io_uring_register +428 common open_tree sys_open_tree +429 common move_mount sys_move_mount +430 common fsopen sys_fsopen +431 common fsconfig sys_fsconfig +432 common fsmount sys_fsmount +433 common fspick sys_fspick +434 common pidfd_open sys_pidfd_open +# 435 reserved for clone3 +436 common close_range sys_close_range +437 common openat2 sys_openat2 +438 common pidfd_getfd sys_pidfd_getfd +439 common faccessat2 sys_faccessat2 +440 common process_madvise sys_process_madvise +441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr +443 common quotactl_fd sys_quotactl_fd +444 common landlock_create_ruleset sys_landlock_create_ruleset +445 common landlock_add_rule sys_landlock_add_rule +446 common landlock_restrict_self sys_landlock_restrict_self +# 447 reserved for memfd_secret +448 common process_mrelease sys_process_mrelease +449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 common cachestat sys_cachestat +452 common fchmodat2 sys_fchmodat2 +453 common map_shadow_stack sys_map_shadow_stack +454 common futex_wake sys_futex_wake +455 common futex_wait sys_futex_wait +456 common futex_requeue sys_futex_requeue +457 common statmount sys_statmount +458 common listmount sys_listmount +459 common lsm_get_self_attr sys_lsm_get_self_attr +460 common lsm_set_self_attr sys_lsm_set_self_attr +461 common lsm_list_modules sys_lsm_list_modules +462 common mseal sys_mseal +463 common setxattrat sys_setxattrat +464 common getxattrat sys_getxattrat +465 common listxattrat sys_listxattrat +466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr +468 common file_getattr sys_file_getattr +469 common file_setattr sys_file_setattr +470 common listns sys_listns diff --git a/arch/sparc/kernel/sysfs.c b/arch/sparc/kernel/sysfs.c index 654e8aad3bbe..f19487e4cc71 100644 --- a/arch/sparc/kernel/sysfs.c +++ b/arch/sparc/kernel/sysfs.c @@ -1,4 +1,5 @@ -/* sysfs.c: Toplogy sysfs support code for sparc64. +// SPDX-License-Identifier: GPL-2.0 +/* sysfs.c: Topology sysfs support code for sparc64. * * Copyright (C) 2007 David S. Miller <davem@davemloft.net> */ @@ -98,27 +99,7 @@ static struct attribute_group mmu_stat_group = { .name = "mmu_stats", }; -/* XXX convert to rusty's on_one_cpu */ -static unsigned long run_on_cpu(unsigned long cpu, - unsigned long (*func)(unsigned long), - unsigned long arg) -{ - cpumask_t old_affinity; - unsigned long ret; - - cpumask_copy(&old_affinity, tsk_cpus_allowed(current)); - /* should return -EINVAL to userspace */ - if (set_cpus_allowed_ptr(current, cpumask_of(cpu))) - return 0; - - ret = func(arg); - - set_cpus_allowed_ptr(current, &old_affinity); - - return ret; -} - -static unsigned long read_mmustat_enable(unsigned long junk) +static long read_mmustat_enable(void *data __maybe_unused) { unsigned long ra = 0; @@ -127,11 +108,11 @@ static unsigned long read_mmustat_enable(unsigned long junk) return ra != 0; } -static unsigned long write_mmustat_enable(unsigned long val) +static long write_mmustat_enable(void *data) { - unsigned long ra, orig_ra; + unsigned long ra, orig_ra, *val = data; - if (val) + if (*val) ra = __pa(&per_cpu(mmu_stats, smp_processor_id())); else ra = 0UL; @@ -142,7 +123,8 @@ static unsigned long write_mmustat_enable(unsigned long val) static ssize_t show_mmustat_enable(struct device *s, struct device_attribute *attr, char *buf) { - unsigned long val = run_on_cpu(s->id, read_mmustat_enable, 0); + long val = work_on_cpu(s->id, read_mmustat_enable, NULL); + return sprintf(buf, "%lx\n", val); } @@ -150,13 +132,15 @@ static ssize_t store_mmustat_enable(struct device *s, struct device_attribute *attr, const char *buf, size_t count) { - unsigned long val, err; - int ret = sscanf(buf, "%ld", &val); + unsigned long val; + long err; + int ret; + ret = sscanf(buf, "%lu", &val); if (ret != 1) return -EINVAL; - err = run_on_cpu(s->id, write_mmustat_enable, val); + err = work_on_cpu(s->id, write_mmustat_enable, &val); if (err) return -EIO; @@ -221,7 +205,7 @@ static struct device_attribute cpu_core_attrs[] = { static DEFINE_PER_CPU(struct cpu, cpu_devices); -static void register_cpu_online(unsigned int cpu) +static int register_cpu_online(unsigned int cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); struct device *s = &c->dev; @@ -231,11 +215,12 @@ static void register_cpu_online(unsigned int cpu) device_create_file(s, &cpu_core_attrs[i]); register_mmu_stats(s); + return 0; } -#ifdef CONFIG_HOTPLUG_CPU -static void unregister_cpu_online(unsigned int cpu) +static int unregister_cpu_online(unsigned int cpu) { +#ifdef CONFIG_HOTPLUG_CPU struct cpu *c = &per_cpu(cpu_devices, cpu); struct device *s = &c->dev; int i; @@ -243,33 +228,10 @@ static void unregister_cpu_online(unsigned int cpu) unregister_mmu_stats(s); for (i = 0; i < ARRAY_SIZE(cpu_core_attrs); i++) device_remove_file(s, &cpu_core_attrs[i]); -} -#endif - -static int __cpuinit sysfs_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned int)(long)hcpu; - - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - register_cpu_online(cpu); - break; -#ifdef CONFIG_HOTPLUG_CPU - case CPU_DEAD: - case CPU_DEAD_FROZEN: - unregister_cpu_online(cpu); - break; #endif - } - return NOTIFY_OK; + return 0; } -static struct notifier_block __cpuinitdata sysfs_cpu_nb = { - .notifier_call = sysfs_cpu_notify, -}; - static void __init check_mmu_stats(void) { unsigned long dummy1, err; @@ -282,34 +244,21 @@ static void __init check_mmu_stats(void) mmu_stats_supported = 1; } -static void register_nodes(void) -{ -#ifdef CONFIG_NUMA - int i; - - for (i = 0; i < MAX_NUMNODES; i++) - register_one_node(i); -#endif -} - static int __init topology_init(void) { - int cpu; - - register_nodes(); + int cpu, ret; check_mmu_stats(); - register_cpu_notifier(&sysfs_cpu_nb); - for_each_possible_cpu(cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); register_cpu(c, cpu); - if (cpu_online(cpu)) - register_cpu_online(cpu); } + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "sparc/topology:online", + register_cpu_online, unregister_cpu_online); + WARN_ON(ret < 0); return 0; } diff --git a/arch/sparc/kernel/systbls.h b/arch/sparc/kernel/systbls.h index 26e6dd72e92a..bf014267d619 100644 --- a/arch/sparc/kernel/systbls.h +++ b/arch/sparc/kernel/systbls.h @@ -1,41 +1,103 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SYSTBLS_H #define _SYSTBLS_H +#include <linux/signal.h> #include <linux/kernel.h> +#include <linux/compat.h> #include <linux/types.h> -#include <linux/signal.h> + #include <asm/utrap.h> -extern asmlinkage unsigned long sys_getpagesize(void); -extern asmlinkage long sparc_pipe(struct pt_regs *regs); -extern asmlinkage long sys_sparc_ipc(unsigned int call, int first, - unsigned long second, - unsigned long third, - void __user *ptr, long fifth); -extern asmlinkage long sparc64_personality(unsigned long personality); -extern asmlinkage long sys64_munmap(unsigned long addr, size_t len); -extern asmlinkage unsigned long sys64_mremap(unsigned long addr, - unsigned long old_len, - unsigned long new_len, - unsigned long flags, - unsigned long new_addr); -extern asmlinkage unsigned long c_sys_nis_syscall(struct pt_regs *regs); -extern asmlinkage long sys_getdomainname(char __user *name, int len); -extern asmlinkage long sys_utrap_install(utrap_entry_t type, - utrap_handler_t new_p, - utrap_handler_t new_d, - utrap_handler_t __user *old_p, - utrap_handler_t __user *old_d); -extern asmlinkage long sparc_memory_ordering(unsigned long model, - struct pt_regs *regs); -extern asmlinkage long sys_rt_sigaction(int sig, - const struct sigaction __user *act, - struct sigaction __user *oact, - void __user *restorer, - size_t sigsetsize); +asmlinkage long sys_getpagesize(void); +asmlinkage long sys_sparc_pipe(void); +asmlinkage long sys_nis_syscall(void); +asmlinkage long sys_getdomainname(char __user *name, int len); +void do_rt_sigreturn(struct pt_regs *regs); +asmlinkage long sys_mmap(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, + unsigned long fd, unsigned long off); +asmlinkage void sparc_breakpoint(struct pt_regs *regs); + +#ifdef CONFIG_SPARC32 +asmlinkage long sys_mmap2(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, + unsigned long fd, unsigned long pgoff); +long sys_sparc_remap_file_pages(unsigned long start, unsigned long size, + unsigned long prot, unsigned long pgoff, + unsigned long flags); -extern asmlinkage void sparc64_set_context(struct pt_regs *regs); -extern asmlinkage void sparc64_get_context(struct pt_regs *regs); -extern void do_rt_sigreturn(struct pt_regs *regs); +#endif /* CONFIG_SPARC32 */ +#ifdef CONFIG_SPARC64 +asmlinkage long sys_sparc_ipc(unsigned int call, int first, + unsigned long second, + unsigned long third, + void __user *ptr, long fifth); +asmlinkage long sparc64_personality(unsigned long personality); +asmlinkage long sys64_munmap(unsigned long addr, size_t len); +asmlinkage unsigned long sys64_mremap(unsigned long addr, + unsigned long old_len, + unsigned long new_len, + unsigned long flags, + unsigned long new_addr); +asmlinkage long sys_utrap_install(utrap_entry_t type, + utrap_handler_t new_p, + utrap_handler_t new_d, + utrap_handler_t __user *old_p, + utrap_handler_t __user *old_d); +asmlinkage long sys_memory_ordering(unsigned long model); +asmlinkage void sparc64_set_context(struct pt_regs *regs); +asmlinkage void sparc64_get_context(struct pt_regs *regs); +asmlinkage long compat_sys_truncate64(const char __user * path, + u32 high, + u32 low); +asmlinkage long compat_sys_ftruncate64(unsigned int fd, + u32 high, + u32 low); +struct compat_stat64; +asmlinkage long compat_sys_stat64(const char __user * filename, + struct compat_stat64 __user *statbuf); +asmlinkage long compat_sys_lstat64(const char __user * filename, + struct compat_stat64 __user *statbuf); +asmlinkage long compat_sys_fstat64(unsigned int fd, + struct compat_stat64 __user * statbuf); +asmlinkage long compat_sys_fstatat64(unsigned int dfd, + const char __user *filename, + struct compat_stat64 __user * statbuf, int flag); +asmlinkage long compat_sys_pread64(unsigned int fd, + char __user *ubuf, + compat_size_t count, + u32 poshi, + u32 poslo); +asmlinkage long compat_sys_pwrite64(unsigned int fd, + char __user *ubuf, + compat_size_t count, + u32 poshi, + u32 poslo); +asmlinkage long compat_sys_readahead(int fd, + unsigned offhi, + unsigned offlo, + compat_size_t count); +long compat_sys_fadvise64(int fd, + unsigned offhi, + unsigned offlo, + compat_size_t len, int advice); +long compat_sys_fadvise64_64(int fd, + unsigned offhi, unsigned offlo, + unsigned lenhi, unsigned lenlo, + int advice); +long compat_sys_sync_file_range(unsigned int fd, + unsigned off_high, unsigned off_low, + unsigned nb_high, unsigned nb_low, + unsigned int flags); +asmlinkage long compat_sys_fallocate(int fd, int mode, u32 offhi, u32 offlo, + u32 lenhi, u32 lenlo); +asmlinkage long compat_sys_fstat64(unsigned int fd, + struct compat_stat64 __user * statbuf); +asmlinkage long compat_sys_fstatat64(unsigned int dfd, + const char __user *filename, + struct compat_stat64 __user * statbuf, + int flag); +#endif /* CONFIG_SPARC64 */ #endif /* _SYSTBLS_H */ diff --git a/arch/sparc/kernel/systbls_32.S b/arch/sparc/kernel/systbls_32.S index 7b87171ecf1e..3aaffa017706 100644 --- a/arch/sparc/kernel/systbls_32.S +++ b/arch/sparc/kernel/systbls_32.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* systbls.S: System call entry point tables for OS compatibility. * The native Linux system call table lives here also. * @@ -8,81 +9,10 @@ * Copyright (C) 1995 Adrian M. Rodriguez (adrian@remus.rutgers.edu) */ - +#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) +#define __SYSCALL(nr, entry) .long entry .data .align 4 - - /* First, the Linux native syscall table. */ - .globl sys_call_table sys_call_table: -/*0*/ .long sys_restart_syscall, sys_exit, sys_fork, sys_read, sys_write -/*5*/ .long sys_open, sys_close, sys_wait4, sys_creat, sys_link -/*10*/ .long sys_unlink, sunos_execv, sys_chdir, sys_chown16, sys_mknod -/*15*/ .long sys_chmod, sys_lchown16, sys_brk, sys_nis_syscall, sys_lseek -/*20*/ .long sys_getpid, sys_capget, sys_capset, sys_setuid16, sys_getuid16 -/*25*/ .long sys_vmsplice, sys_ptrace, sys_alarm, sys_sigaltstack, sys_pause -/*30*/ .long sys_utime, sys_lchown, sys_fchown, sys_access, sys_nice -/*35*/ .long sys_chown, sys_sync, sys_kill, sys_newstat, sys_sendfile -/*40*/ .long sys_newlstat, sys_dup, sys_sparc_pipe, sys_times, sys_getuid -/*45*/ .long sys_umount, sys_setgid16, sys_getgid16, sys_signal, sys_geteuid16 -/*50*/ .long sys_getegid16, sys_acct, sys_nis_syscall, sys_getgid, sys_ioctl -/*55*/ .long sys_reboot, sys_mmap2, sys_symlink, sys_readlink, sys_execve -/*60*/ .long sys_umask, sys_chroot, sys_newfstat, sys_fstat64, sys_getpagesize -/*65*/ .long sys_msync, sys_vfork, sys_pread64, sys_pwrite64, sys_geteuid -/*70*/ .long sys_getegid, sys_mmap, sys_setreuid, sys_munmap, sys_mprotect -/*75*/ .long sys_madvise, sys_vhangup, sys_truncate64, sys_mincore, sys_getgroups16 -/*80*/ .long sys_setgroups16, sys_getpgrp, sys_setgroups, sys_setitimer, sys_ftruncate64 -/*85*/ .long sys_swapon, sys_getitimer, sys_setuid, sys_sethostname, sys_setgid -/*90*/ .long sys_dup2, sys_setfsuid, sys_fcntl, sys_select, sys_setfsgid -/*95*/ .long sys_fsync, sys_setpriority, sys_nis_syscall, sys_nis_syscall, sys_nis_syscall -/*100*/ .long sys_getpriority, sys_rt_sigreturn, sys_rt_sigaction, sys_rt_sigprocmask, sys_rt_sigpending -/*105*/ .long sys_rt_sigtimedwait, sys_rt_sigqueueinfo, sys_rt_sigsuspend, sys_setresuid, sys_getresuid -/*110*/ .long sys_setresgid, sys_getresgid, sys_setregid, sys_nis_syscall, sys_nis_syscall -/*115*/ .long sys_getgroups, sys_gettimeofday, sys_getrusage, sys_nis_syscall, sys_getcwd -/*120*/ .long sys_readv, sys_writev, sys_settimeofday, sys_fchown16, sys_fchmod -/*125*/ .long sys_nis_syscall, sys_setreuid16, sys_setregid16, sys_rename, sys_truncate -/*130*/ .long sys_ftruncate, sys_flock, sys_lstat64, sys_nis_syscall, sys_nis_syscall -/*135*/ .long sys_nis_syscall, sys_mkdir, sys_rmdir, sys_utimes, sys_stat64 -/*140*/ .long sys_sendfile64, sys_nis_syscall, sys_futex, sys_gettid, sys_getrlimit -/*145*/ .long sys_setrlimit, sys_pivot_root, sys_prctl, sys_pciconfig_read, sys_pciconfig_write -/*150*/ .long sys_nis_syscall, sys_inotify_init, sys_inotify_add_watch, sys_poll, sys_getdents64 -/*155*/ .long sys_fcntl64, sys_inotify_rm_watch, sys_statfs, sys_fstatfs, sys_oldumount -/*160*/ .long sys_sched_setaffinity, sys_sched_getaffinity, sys_getdomainname, sys_setdomainname, sys_nis_syscall -/*165*/ .long sys_quotactl, sys_set_tid_address, sys_mount, sys_ustat, sys_setxattr -/*170*/ .long sys_lsetxattr, sys_fsetxattr, sys_getxattr, sys_lgetxattr, sys_getdents -/*175*/ .long sys_setsid, sys_fchdir, sys_fgetxattr, sys_listxattr, sys_llistxattr -/*180*/ .long sys_flistxattr, sys_removexattr, sys_lremovexattr, sys_sigpending, sys_ni_syscall -/*185*/ .long sys_setpgid, sys_fremovexattr, sys_tkill, sys_exit_group, sys_newuname -/*190*/ .long sys_init_module, sys_personality, sparc_remap_file_pages, sys_epoll_create, sys_epoll_ctl -/*195*/ .long sys_epoll_wait, sys_ioprio_set, sys_getppid, sys_sparc_sigaction, sys_sgetmask -/*200*/ .long sys_ssetmask, sys_sigsuspend, sys_newlstat, sys_uselib, sys_old_readdir -/*205*/ .long sys_readahead, sys_socketcall, sys_syslog, sys_lookup_dcookie, sys_fadvise64 -/*210*/ .long sys_fadvise64_64, sys_tgkill, sys_waitpid, sys_swapoff, sys_sysinfo -/*215*/ .long sys_ipc, sys_sigreturn, sys_clone, sys_ioprio_get, sys_adjtimex -/*220*/ .long sys_sigprocmask, sys_ni_syscall, sys_delete_module, sys_ni_syscall, sys_getpgid -/*225*/ .long sys_bdflush, sys_sysfs, sys_nis_syscall, sys_setfsuid16, sys_setfsgid16 -/*230*/ .long sys_select, sys_time, sys_splice, sys_stime, sys_statfs64 - /* "We are the Knights of the Forest of Ni!!" */ -/*235*/ .long sys_fstatfs64, sys_llseek, sys_mlock, sys_munlock, sys_mlockall -/*240*/ .long sys_munlockall, sys_sched_setparam, sys_sched_getparam, sys_sched_setscheduler, sys_sched_getscheduler -/*245*/ .long sys_sched_yield, sys_sched_get_priority_max, sys_sched_get_priority_min, sys_sched_rr_get_interval, sys_nanosleep -/*250*/ .long sys_mremap, sys_sysctl, sys_getsid, sys_fdatasync, sys_ni_syscall -/*255*/ .long sys_sync_file_range, sys_clock_settime, sys_clock_gettime, sys_clock_getres, sys_clock_nanosleep -/*260*/ .long sys_sched_getaffinity, sys_sched_setaffinity, sys_timer_settime, sys_timer_gettime, sys_timer_getoverrun -/*265*/ .long sys_timer_delete, sys_timer_create, sys_nis_syscall, sys_io_setup, sys_io_destroy -/*270*/ .long sys_io_submit, sys_io_cancel, sys_io_getevents, sys_mq_open, sys_mq_unlink -/*275*/ .long sys_mq_timedsend, sys_mq_timedreceive, sys_mq_notify, sys_mq_getsetattr, sys_waitid -/*280*/ .long sys_tee, sys_add_key, sys_request_key, sys_keyctl, sys_openat -/*285*/ .long sys_mkdirat, sys_mknodat, sys_fchownat, sys_futimesat, sys_fstatat64 -/*290*/ .long sys_unlinkat, sys_renameat, sys_linkat, sys_symlinkat, sys_readlinkat -/*295*/ .long sys_fchmodat, sys_faccessat, sys_pselect6, sys_ppoll, sys_unshare -/*300*/ .long sys_set_robust_list, sys_get_robust_list, sys_migrate_pages, sys_mbind, sys_get_mempolicy -/*305*/ .long sys_set_mempolicy, sys_kexec_load, sys_move_pages, sys_getcpu, sys_epoll_pwait -/*310*/ .long sys_utimensat, sys_signalfd, sys_timerfd_create, sys_eventfd, sys_fallocate -/*315*/ .long sys_timerfd_settime, sys_timerfd_gettime, sys_signalfd4, sys_eventfd2, sys_epoll_create1 -/*320*/ .long sys_dup3, sys_pipe2, sys_inotify_init1, sys_accept4, sys_preadv -/*325*/ .long sys_pwritev, sys_rt_tgsigqueueinfo, sys_perf_event_open, sys_recvmmsg, sys_fanotify_init -/*330*/ .long sys_fanotify_mark, sys_prlimit64, sys_name_to_handle_at, sys_open_by_handle_at, sys_clock_adjtime -/*335*/ .long sys_syncfs, sys_sendmmsg, sys_setns, sys_process_vm_readv, sys_process_vm_writev -/*340*/ .long sys_ni_syscall, sys_kcmp, sys_finit_module +#include <asm/syscall_table_32.h> /* 32-bit native syscalls */ diff --git a/arch/sparc/kernel/systbls_64.S b/arch/sparc/kernel/systbls_64.S index 6d81597064b6..398fe449dd34 100644 --- a/arch/sparc/kernel/systbls_64.S +++ b/arch/sparc/kernel/systbls_64.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* systbls.S: System call entry point tables for OS compatibility. * The native Linux system call table lives here also. * @@ -9,159 +10,20 @@ * Copyright (C) 1995 Adrian M. Rodriguez (adrian@remus.rutgers.edu) */ - +#define __SYSCALL(nr, entry) .word entry .text .align 4 - #ifdef CONFIG_COMPAT - /* First, the 32-bit Linux native syscall table. */ - .globl sys_call_table32 sys_call_table32: -/*0*/ .word sys_restart_syscall, sparc_exit, sys_fork, sys_read, sys_write -/*5*/ .word compat_sys_open, sys_close, compat_sys_wait4, sys_creat, sys_link -/*10*/ .word sys_unlink, sunos_execv, sys_chdir, sys_chown16, sys_mknod -/*15*/ .word sys_chmod, sys_lchown16, sys_brk, sys_nis_syscall, compat_sys_lseek -/*20*/ .word sys_getpid, sys_capget, sys_capset, sys_setuid16, sys_getuid16 -/*25*/ .word compat_sys_vmsplice, compat_sys_ptrace, sys_alarm, compat_sys_sigaltstack, sys_pause -/*30*/ .word compat_sys_utime, sys_lchown, sys_fchown, sys_access, sys_nice - .word sys_chown, sys_sync, sys_kill, compat_sys_newstat, compat_sys_sendfile -/*40*/ .word compat_sys_newlstat, sys_dup, sys_sparc_pipe, compat_sys_times, sys_getuid - .word sys_umount, sys_setgid16, sys_getgid16, sys_signal, sys_geteuid16 -/*50*/ .word sys_getegid16, sys_acct, sys_nis_syscall, sys_getgid, compat_sys_ioctl - .word sys_reboot, sys32_mmap2, sys_symlink, sys_readlink, sys32_execve -/*60*/ .word sys_umask, sys_chroot, compat_sys_newfstat, compat_sys_fstat64, sys_getpagesize - .word sys_msync, sys_vfork, sys32_pread64, sys32_pwrite64, sys_geteuid -/*70*/ .word sys_getegid, sys_mmap, sys_setreuid, sys_munmap, sys_mprotect - .word sys_madvise, sys_vhangup, sys32_truncate64, sys_mincore, sys_getgroups16 -/*80*/ .word sys_setgroups16, sys_getpgrp, sys_setgroups, compat_sys_setitimer, sys32_ftruncate64 - .word sys_swapon, compat_sys_getitimer, sys_setuid, sys_sethostname, sys_setgid -/*90*/ .word sys_dup2, sys_setfsuid, compat_sys_fcntl, sys32_select, sys_setfsgid - .word sys_fsync, sys_setpriority, sys_nis_syscall, sys_nis_syscall, sys_nis_syscall -/*100*/ .word sys_getpriority, sys32_rt_sigreturn, compat_sys_rt_sigaction, compat_sys_rt_sigprocmask, compat_sys_rt_sigpending - .word compat_sys_rt_sigtimedwait, compat_sys_rt_sigqueueinfo, compat_sys_rt_sigsuspend, sys_setresuid, sys_getresuid -/*110*/ .word sys_setresgid, sys_getresgid, sys_setregid, sys_nis_syscall, sys_nis_syscall - .word sys_getgroups, compat_sys_gettimeofday, compat_sys_getrusage, sys_nis_syscall, sys_getcwd -/*120*/ .word compat_sys_readv, compat_sys_writev, compat_sys_settimeofday, sys_fchown16, sys_fchmod - .word sys_nis_syscall, sys_setreuid16, sys_setregid16, sys_rename, compat_sys_truncate -/*130*/ .word compat_sys_ftruncate, sys_flock, compat_sys_lstat64, sys_nis_syscall, sys_nis_syscall - .word sys_nis_syscall, sys_mkdir, sys_rmdir, compat_sys_utimes, compat_sys_stat64 -/*140*/ .word sys_sendfile64, sys_nis_syscall, sys32_futex, sys_gettid, compat_sys_getrlimit - .word compat_sys_setrlimit, sys_pivot_root, sys_prctl, sys_pciconfig_read, sys_pciconfig_write -/*150*/ .word sys_nis_syscall, sys_inotify_init, sys_inotify_add_watch, sys_poll, sys_getdents64 - .word compat_sys_fcntl64, sys_inotify_rm_watch, compat_sys_statfs, compat_sys_fstatfs, sys_oldumount -/*160*/ .word compat_sys_sched_setaffinity, compat_sys_sched_getaffinity, sys_getdomainname, sys_setdomainname, sys_nis_syscall - .word sys_quotactl, sys_set_tid_address, compat_sys_mount, compat_sys_ustat, sys_setxattr -/*170*/ .word sys_lsetxattr, sys_fsetxattr, sys_getxattr, sys_lgetxattr, compat_sys_getdents - .word sys_setsid, sys_fchdir, sys_fgetxattr, sys_listxattr, sys_llistxattr -/*180*/ .word sys_flistxattr, sys_removexattr, sys_lremovexattr, compat_sys_sigpending, sys_ni_syscall - .word sys_setpgid, sys_fremovexattr, sys_tkill, sparc_exit_group, sys_newuname -/*190*/ .word sys_init_module, sys_sparc64_personality, sys_remap_file_pages, sys_epoll_create, sys_epoll_ctl - .word sys_epoll_wait, sys_ioprio_set, sys_getppid, compat_sys_sparc_sigaction, sys_sgetmask -/*200*/ .word sys_ssetmask, sys_sigsuspend, compat_sys_newlstat, sys_uselib, compat_sys_old_readdir - .word sys32_readahead, sys32_socketcall, sys_syslog, compat_sys_lookup_dcookie, sys32_fadvise64 -/*210*/ .word sys32_fadvise64_64, sys_tgkill, sys_waitpid, sys_swapoff, compat_sys_sysinfo - .word compat_sys_ipc, sys32_sigreturn, sys_clone, sys_ioprio_get, compat_sys_adjtimex -/*220*/ .word compat_sys_sigprocmask, sys_ni_syscall, sys_delete_module, sys_ni_syscall, sys_getpgid - .word sys_bdflush, sys_sysfs, sys_nis_syscall, sys_setfsuid16, sys_setfsgid16 -/*230*/ .word sys32_select, compat_sys_time, sys_splice, compat_sys_stime, compat_sys_statfs64 - .word compat_sys_fstatfs64, sys_llseek, sys_mlock, sys_munlock, sys_mlockall -/*240*/ .word sys_munlockall, sys_sched_setparam, sys_sched_getparam, sys_sched_setscheduler, sys_sched_getscheduler - .word sys_sched_yield, sys_sched_get_priority_max, sys_sched_get_priority_min, compat_sys_sched_rr_get_interval, compat_sys_nanosleep -/*250*/ .word sys_mremap, compat_sys_sysctl, sys_getsid, sys_fdatasync, sys_nis_syscall - .word sys32_sync_file_range, compat_sys_clock_settime, compat_sys_clock_gettime, compat_sys_clock_getres, sys32_clock_nanosleep -/*260*/ .word compat_sys_sched_getaffinity, compat_sys_sched_setaffinity, sys32_timer_settime, compat_sys_timer_gettime, sys_timer_getoverrun - .word sys_timer_delete, compat_sys_timer_create, sys_ni_syscall, compat_sys_io_setup, sys_io_destroy -/*270*/ .word sys32_io_submit, sys_io_cancel, compat_sys_io_getevents, sys32_mq_open, sys_mq_unlink - .word compat_sys_mq_timedsend, compat_sys_mq_timedreceive, compat_sys_mq_notify, compat_sys_mq_getsetattr, compat_sys_waitid -/*280*/ .word sys_tee, sys_add_key, sys_request_key, compat_sys_keyctl, compat_sys_openat - .word sys_mkdirat, sys_mknodat, sys_fchownat, compat_sys_futimesat, compat_sys_fstatat64 -/*290*/ .word sys_unlinkat, sys_renameat, sys_linkat, sys_symlinkat, sys_readlinkat - .word sys_fchmodat, sys_faccessat, compat_sys_pselect6, compat_sys_ppoll, sys_unshare -/*300*/ .word compat_sys_set_robust_list, compat_sys_get_robust_list, compat_sys_migrate_pages, compat_sys_mbind, compat_sys_get_mempolicy - .word compat_sys_set_mempolicy, compat_sys_kexec_load, compat_sys_move_pages, sys_getcpu, compat_sys_epoll_pwait -/*310*/ .word compat_sys_utimensat, compat_sys_signalfd, sys_timerfd_create, sys_eventfd, compat_sys_fallocate - .word compat_sys_timerfd_settime, compat_sys_timerfd_gettime, compat_sys_signalfd4, sys_eventfd2, sys_epoll_create1 -/*320*/ .word sys_dup3, sys_pipe2, sys_inotify_init1, sys_accept4, compat_sys_preadv - .word compat_sys_pwritev, compat_sys_rt_tgsigqueueinfo, sys_perf_event_open, compat_sys_recvmmsg, sys_fanotify_init -/*330*/ .word compat_sys_fanotify_mark, sys_prlimit64, sys_name_to_handle_at, compat_sys_open_by_handle_at, compat_sys_clock_adjtime - .word sys_syncfs, compat_sys_sendmmsg, sys_setns, compat_sys_process_vm_readv, compat_sys_process_vm_writev -/*340*/ .word sys_kern_features, sys_kcmp, sys_finit_module - +#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, compat) +#include <asm/syscall_table_32.h> /* Compat syscalls */ +#undef __SYSCALL_WITH_COMPAT #endif /* CONFIG_COMPAT */ - /* Now the 64-bit native Linux syscall table. */ - .align 4 .globl sys_call_table64, sys_call_table sys_call_table64: sys_call_table: -/*0*/ .word sys_restart_syscall, sparc_exit, sys_fork, sys_read, sys_write -/*5*/ .word sys_open, sys_close, sys_wait4, sys_creat, sys_link -/*10*/ .word sys_unlink, sys_nis_syscall, sys_chdir, sys_chown, sys_mknod -/*15*/ .word sys_chmod, sys_lchown, sys_brk, sys_nis_syscall, sys_lseek -/*20*/ .word sys_getpid, sys_capget, sys_capset, sys_setuid, sys_getuid -/*25*/ .word sys_vmsplice, sys_ptrace, sys_alarm, sys_sigaltstack, sys_nis_syscall -/*30*/ .word sys_utime, sys_nis_syscall, sys_nis_syscall, sys_access, sys_nice - .word sys_nis_syscall, sys_sync, sys_kill, sys_newstat, sys_sendfile64 -/*40*/ .word sys_newlstat, sys_dup, sys_sparc_pipe, sys_times, sys_nis_syscall - .word sys_umount, sys_setgid, sys_getgid, sys_signal, sys_geteuid -/*50*/ .word sys_getegid, sys_acct, sys_memory_ordering, sys_nis_syscall, sys_ioctl - .word sys_reboot, sys_nis_syscall, sys_symlink, sys_readlink, sys64_execve -/*60*/ .word sys_umask, sys_chroot, sys_newfstat, sys_fstat64, sys_getpagesize - .word sys_msync, sys_vfork, sys_pread64, sys_pwrite64, sys_nis_syscall -/*70*/ .word sys_nis_syscall, sys_mmap, sys_nis_syscall, sys_64_munmap, sys_mprotect - .word sys_madvise, sys_vhangup, sys_nis_syscall, sys_mincore, sys_getgroups -/*80*/ .word sys_setgroups, sys_getpgrp, sys_nis_syscall, sys_setitimer, sys_nis_syscall - .word sys_swapon, sys_getitimer, sys_nis_syscall, sys_sethostname, sys_nis_syscall -/*90*/ .word sys_dup2, sys_nis_syscall, sys_fcntl, sys_select, sys_nis_syscall - .word sys_fsync, sys_setpriority, sys_socket, sys_connect, sys_accept -/*100*/ .word sys_getpriority, sys_rt_sigreturn, sys_rt_sigaction, sys_rt_sigprocmask, sys_rt_sigpending - .word sys_rt_sigtimedwait, sys_rt_sigqueueinfo, sys_rt_sigsuspend, sys_setresuid, sys_getresuid -/*110*/ .word sys_setresgid, sys_getresgid, sys_nis_syscall, sys_recvmsg, sys_sendmsg - .word sys_nis_syscall, sys_gettimeofday, sys_getrusage, sys_getsockopt, sys_getcwd -/*120*/ .word sys_readv, sys_writev, sys_settimeofday, sys_fchown, sys_fchmod - .word sys_recvfrom, sys_setreuid, sys_setregid, sys_rename, sys_truncate -/*130*/ .word sys_ftruncate, sys_flock, sys_lstat64, sys_sendto, sys_shutdown - .word sys_socketpair, sys_mkdir, sys_rmdir, sys_utimes, sys_stat64 -/*140*/ .word sys_sendfile64, sys_getpeername, sys_futex, sys_gettid, sys_getrlimit - .word sys_setrlimit, sys_pivot_root, sys_prctl, sys_pciconfig_read, sys_pciconfig_write -/*150*/ .word sys_getsockname, sys_inotify_init, sys_inotify_add_watch, sys_poll, sys_getdents64 - .word sys_nis_syscall, sys_inotify_rm_watch, sys_statfs, sys_fstatfs, sys_oldumount -/*160*/ .word sys_sched_setaffinity, sys_sched_getaffinity, sys_getdomainname, sys_setdomainname, sys_utrap_install - .word sys_quotactl, sys_set_tid_address, sys_mount, sys_ustat, sys_setxattr -/*170*/ .word sys_lsetxattr, sys_fsetxattr, sys_getxattr, sys_lgetxattr, sys_getdents - .word sys_setsid, sys_fchdir, sys_fgetxattr, sys_listxattr, sys_llistxattr -/*180*/ .word sys_flistxattr, sys_removexattr, sys_lremovexattr, sys_nis_syscall, sys_ni_syscall - .word sys_setpgid, sys_fremovexattr, sys_tkill, sparc_exit_group, sys_newuname -/*190*/ .word sys_init_module, sys_sparc64_personality, sys_remap_file_pages, sys_epoll_create, sys_epoll_ctl - .word sys_epoll_wait, sys_ioprio_set, sys_getppid, sys_nis_syscall, sys_sgetmask -/*200*/ .word sys_ssetmask, sys_nis_syscall, sys_newlstat, sys_uselib, sys_nis_syscall - .word sys_readahead, sys_socketcall, sys_syslog, sys_lookup_dcookie, sys_fadvise64 -/*210*/ .word sys_fadvise64_64, sys_tgkill, sys_waitpid, sys_swapoff, sys_sysinfo - .word sys_sparc_ipc, sys_nis_syscall, sys_clone, sys_ioprio_get, sys_adjtimex -/*220*/ .word sys_nis_syscall, sys_ni_syscall, sys_delete_module, sys_ni_syscall, sys_getpgid - .word sys_bdflush, sys_sysfs, sys_nis_syscall, sys_setfsuid, sys_setfsgid -/*230*/ .word sys_select, sys_nis_syscall, sys_splice, sys_stime, sys_statfs64 - .word sys_fstatfs64, sys_llseek, sys_mlock, sys_munlock, sys_mlockall -/*240*/ .word sys_munlockall, sys_sched_setparam, sys_sched_getparam, sys_sched_setscheduler, sys_sched_getscheduler - .word sys_sched_yield, sys_sched_get_priority_max, sys_sched_get_priority_min, sys_sched_rr_get_interval, sys_nanosleep -/*250*/ .word sys_64_mremap, sys_sysctl, sys_getsid, sys_fdatasync, sys_nis_syscall - .word sys_sync_file_range, sys_clock_settime, sys_clock_gettime, sys_clock_getres, sys_clock_nanosleep -/*260*/ .word sys_sched_getaffinity, sys_sched_setaffinity, sys_timer_settime, sys_timer_gettime, sys_timer_getoverrun - .word sys_timer_delete, sys_timer_create, sys_ni_syscall, sys_io_setup, sys_io_destroy -/*270*/ .word sys_io_submit, sys_io_cancel, sys_io_getevents, sys_mq_open, sys_mq_unlink - .word sys_mq_timedsend, sys_mq_timedreceive, sys_mq_notify, sys_mq_getsetattr, sys_waitid -/*280*/ .word sys_tee, sys_add_key, sys_request_key, sys_keyctl, sys_openat - .word sys_mkdirat, sys_mknodat, sys_fchownat, sys_futimesat, sys_fstatat64 -/*290*/ .word sys_unlinkat, sys_renameat, sys_linkat, sys_symlinkat, sys_readlinkat - .word sys_fchmodat, sys_faccessat, sys_pselect6, sys_ppoll, sys_unshare -/*300*/ .word sys_set_robust_list, sys_get_robust_list, sys_migrate_pages, sys_mbind, sys_get_mempolicy - .word sys_set_mempolicy, sys_kexec_load, sys_move_pages, sys_getcpu, sys_epoll_pwait -/*310*/ .word sys_utimensat, sys_signalfd, sys_timerfd_create, sys_eventfd, sys_fallocate - .word sys_timerfd_settime, sys_timerfd_gettime, sys_signalfd4, sys_eventfd2, sys_epoll_create1 -/*320*/ .word sys_dup3, sys_pipe2, sys_inotify_init1, sys_accept4, sys_preadv - .word sys_pwritev, sys_rt_tgsigqueueinfo, sys_perf_event_open, sys_recvmmsg, sys_fanotify_init -/*330*/ .word sys_fanotify_mark, sys_prlimit64, sys_name_to_handle_at, sys_open_by_handle_at, sys_clock_adjtime - .word sys_syncfs, sys_sendmmsg, sys_setns, sys_process_vm_readv, sys_process_vm_writev -/*340*/ .word sys_kern_features, sys_kcmp, sys_finit_module +#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) +#include <asm/syscall_table_64.h> /* 64-bit native syscalls */ diff --git a/arch/sparc/kernel/tadpole.c b/arch/sparc/kernel/tadpole.c deleted file mode 100644 index 9aba8bd5a78b..000000000000 --- a/arch/sparc/kernel/tadpole.c +++ /dev/null @@ -1,126 +0,0 @@ -/* tadpole.c: Probing for the tadpole clock stopping h/w at boot time. - * - * Copyright (C) 1996 David Redman (djhr@tadpole.co.uk) - */ - -#include <linux/string.h> -#include <linux/kernel.h> -#include <linux/sched.h> -#include <linux/init.h> - -#include <asm/asi.h> -#include <asm/oplib.h> -#include <asm/io.h> - -#define MACIO_SCSI_CSR_ADDR 0x78400000 -#define MACIO_EN_DMA 0x00000200 -#define CLOCK_INIT_DONE 1 - -static int clk_state; -static volatile unsigned char *clk_ctrl; -void (*cpu_pwr_save)(void); - -static inline unsigned int ldphys(unsigned int addr) -{ - unsigned long data; - - __asm__ __volatile__("\n\tlda [%1] %2, %0\n\t" : - "=r" (data) : - "r" (addr), "i" (ASI_M_BYPASS)); - return data; -} - -static void clk_init(void) -{ - __asm__ __volatile__("mov 0x6c, %%g1\n\t" - "mov 0x4c, %%g2\n\t" - "mov 0xdf, %%g3\n\t" - "stb %%g1, [%0+3]\n\t" - "stb %%g2, [%0+3]\n\t" - "stb %%g3, [%0+3]\n\t" : : - "r" (clk_ctrl) : - "g1", "g2", "g3"); -} - -static void clk_slow(void) -{ - __asm__ __volatile__("mov 0xcc, %%g2\n\t" - "mov 0x4c, %%g3\n\t" - "mov 0xcf, %%g4\n\t" - "mov 0xdf, %%g5\n\t" - "stb %%g2, [%0+3]\n\t" - "stb %%g3, [%0+3]\n\t" - "stb %%g4, [%0+3]\n\t" - "stb %%g5, [%0+3]\n\t" : : - "r" (clk_ctrl) : - "g2", "g3", "g4", "g5"); -} - -/* - * Tadpole is guaranteed to be UP, using local_irq_save. - */ -static void tsu_clockstop(void) -{ - unsigned int mcsr; - unsigned long flags; - - if (!clk_ctrl) - return; - if (!(clk_state & CLOCK_INIT_DONE)) { - local_irq_save(flags); - clk_init(); - clk_state |= CLOCK_INIT_DONE; /* all done */ - local_irq_restore(flags); - return; - } - if (!(clk_ctrl[2] & 1)) - return; /* no speed up yet */ - - local_irq_save(flags); - - /* if SCSI DMA in progress, don't slow clock */ - mcsr = ldphys(MACIO_SCSI_CSR_ADDR); - if ((mcsr&MACIO_EN_DMA) != 0) { - local_irq_restore(flags); - return; - } - /* TODO... the minimum clock setting ought to increase the - * memory refresh interval.. - */ - clk_slow(); - local_irq_restore(flags); -} - -static void swift_clockstop(void) -{ - if (!clk_ctrl) - return; - clk_ctrl[0] = 0; -} - -void __init clock_stop_probe(void) -{ - phandle node, clk_nd; - char name[20]; - - prom_getstring(prom_root_node, "name", name, sizeof(name)); - if (strncmp(name, "Tadpole", 7)) - return; - node = prom_getchild(prom_root_node); - node = prom_searchsiblings(node, "obio"); - node = prom_getchild(node); - clk_nd = prom_searchsiblings(node, "clk-ctrl"); - if (!clk_nd) - return; - printk("Clock Stopping h/w detected... "); - clk_ctrl = (char *) prom_getint(clk_nd, "address"); - clk_state = 0; - if (name[10] == '\0') { - cpu_pwr_save = tsu_clockstop; - printk("enabled (S3)\n"); - } else if ((name[10] == 'X') || (name[10] == 'G')) { - cpu_pwr_save = swift_clockstop; - printk("enabled (%s)\n",name+7); - } else - printk("disabled %s\n",name+7); -} diff --git a/arch/sparc/kernel/termios.c b/arch/sparc/kernel/termios.c new file mode 100644 index 000000000000..ee64965c27cd --- /dev/null +++ b/arch/sparc/kernel/termios.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/termios_internal.h> + +/* + * c_cc characters in the termio structure. Oh, how I love being + * backwardly compatible. Notice that character 4 and 5 are + * interpreted differently depending on whether ICANON is set in + * c_lflag. If it's set, they are used as _VEOF and _VEOL, otherwise + * as _VMIN and V_TIME. This is for compatibility with OSF/1 (which + * is compatible with sysV)... + */ +#define _VMIN 4 +#define _VTIME 5 + +int kernel_termios_to_user_termio(struct termio __user *termio, + struct ktermios *termios) +{ + struct termio v; + memset(&v, 0, sizeof(struct termio)); + v.c_iflag = termios->c_iflag; + v.c_oflag = termios->c_oflag; + v.c_cflag = termios->c_cflag; + v.c_lflag = termios->c_lflag; + v.c_line = termios->c_line; + memcpy(v.c_cc, termios->c_cc, NCC); + if (!(v.c_lflag & ICANON)) { + v.c_cc[_VMIN] = termios->c_cc[VMIN]; + v.c_cc[_VTIME] = termios->c_cc[VTIME]; + } + return copy_to_user(termio, &v, sizeof(struct termio)); +} + +int user_termios_to_kernel_termios(struct ktermios *k, + struct termios2 __user *u) +{ + int err; + err = get_user(k->c_iflag, &u->c_iflag); + err |= get_user(k->c_oflag, &u->c_oflag); + err |= get_user(k->c_cflag, &u->c_cflag); + err |= get_user(k->c_lflag, &u->c_lflag); + err |= get_user(k->c_line, &u->c_line); + err |= copy_from_user(k->c_cc, u->c_cc, NCCS); + if (k->c_lflag & ICANON) { + err |= get_user(k->c_cc[VEOF], &u->c_cc[VEOF]); + err |= get_user(k->c_cc[VEOL], &u->c_cc[VEOL]); + } else { + err |= get_user(k->c_cc[VMIN], &u->c_cc[_VMIN]); + err |= get_user(k->c_cc[VTIME], &u->c_cc[_VTIME]); + } + err |= get_user(k->c_ispeed, &u->c_ispeed); + err |= get_user(k->c_ospeed, &u->c_ospeed); + return err; +} + +int kernel_termios_to_user_termios(struct termios2 __user *u, + struct ktermios *k) +{ + int err; + err = put_user(k->c_iflag, &u->c_iflag); + err |= put_user(k->c_oflag, &u->c_oflag); + err |= put_user(k->c_cflag, &u->c_cflag); + err |= put_user(k->c_lflag, &u->c_lflag); + err |= put_user(k->c_line, &u->c_line); + err |= copy_to_user(u->c_cc, k->c_cc, NCCS); + if (!(k->c_lflag & ICANON)) { + err |= put_user(k->c_cc[VMIN], &u->c_cc[_VMIN]); + err |= put_user(k->c_cc[VTIME], &u->c_cc[_VTIME]); + } else { + err |= put_user(k->c_cc[VEOF], &u->c_cc[VEOF]); + err |= put_user(k->c_cc[VEOL], &u->c_cc[VEOL]); + } + err |= put_user(k->c_ispeed, &u->c_ispeed); + err |= put_user(k->c_ospeed, &u->c_ospeed); + return err; +} + +int user_termios_to_kernel_termios_1(struct ktermios *k, + struct termios __user *u) +{ + int err; + err = get_user(k->c_iflag, &u->c_iflag); + err |= get_user(k->c_oflag, &u->c_oflag); + err |= get_user(k->c_cflag, &u->c_cflag); + err |= get_user(k->c_lflag, &u->c_lflag); + err |= get_user(k->c_line, &u->c_line); + err |= copy_from_user(k->c_cc, u->c_cc, NCCS); + if (k->c_lflag & ICANON) { + err |= get_user(k->c_cc[VEOF], &u->c_cc[VEOF]); + err |= get_user(k->c_cc[VEOL], &u->c_cc[VEOL]); + } else { + err |= get_user(k->c_cc[VMIN], &u->c_cc[_VMIN]); + err |= get_user(k->c_cc[VTIME], &u->c_cc[_VTIME]); + } + return err; +} + +int kernel_termios_to_user_termios_1(struct termios __user *u, + struct ktermios *k) +{ + int err; + err = put_user(k->c_iflag, &u->c_iflag); + err |= put_user(k->c_oflag, &u->c_oflag); + err |= put_user(k->c_cflag, &u->c_cflag); + err |= put_user(k->c_lflag, &u->c_lflag); + err |= put_user(k->c_line, &u->c_line); + err |= copy_to_user(u->c_cc, k->c_cc, NCCS); + if (!(k->c_lflag & ICANON)) { + err |= put_user(k->c_cc[VMIN], &u->c_cc[_VMIN]); + err |= put_user(k->c_cc[VTIME], &u->c_cc[_VTIME]); + } else { + err |= put_user(k->c_cc[VEOF], &u->c_cc[VEOF]); + err |= put_user(k->c_cc[VEOL], &u->c_cc[VEOL]); + } + return err; +} diff --git a/arch/sparc/kernel/time_32.c b/arch/sparc/kernel/time_32.c index c4c27b0f9063..578fd0d49f30 100644 --- a/arch/sparc/kernel/time_32.c +++ b/arch/sparc/kernel/time_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* linux/arch/sparc/kernel/time.c * * Copyright (C) 1995 David S. Miller (davem@davemloft.net) @@ -23,7 +24,6 @@ #include <linux/mm.h> #include <linux/interrupt.h> #include <linux/time.h> -#include <linux/rtc.h> #include <linux/rtc/m48t59.h> #include <linux/timex.h> #include <linux/clocksource.h> @@ -33,9 +33,9 @@ #include <linux/ioport.h> #include <linux/profile.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/platform_device.h> +#include <asm/mc146818rtc.h> #include <asm/oplib.h> #include <asm/timex.h> #include <asm/timer.h> @@ -47,6 +47,7 @@ #include <asm/irq_regs.h> #include <asm/setup.h> +#include "kernel.h" #include "irq.h" static __cacheline_aligned_in_smp DEFINE_SEQLOCK(timer_cs_lock); @@ -63,8 +64,6 @@ DEFINE_PER_CPU(struct clock_event_device, sparc32_clockevent); DEFINE_SPINLOCK(rtc_lock); EXPORT_SYMBOL(rtc_lock); -static int set_rtc_mmss(unsigned long); - unsigned long profile_pc(struct pt_regs *regs) { extern char __copy_user_begin[], __copy_user_end[]; @@ -83,12 +82,7 @@ unsigned long profile_pc(struct pt_regs *regs) EXPORT_SYMBOL(profile_pc); -__volatile__ unsigned int *master_l10_counter; - -int update_persistent_clock(struct timespec now) -{ - return set_rtc_mmss(now.tv_sec); -} +volatile u32 __iomem *master_l10_counter; irqreturn_t notrace timer_interrupt(int dummy, void *dev_id) { @@ -107,21 +101,18 @@ irqreturn_t notrace timer_interrupt(int dummy, void *dev_id) return IRQ_HANDLED; } -static void timer_ce_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int timer_ce_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - case CLOCK_EVT_MODE_RESUME: - timer_ce_enabled = 1; - break; - case CLOCK_EVT_MODE_SHUTDOWN: - timer_ce_enabled = 0; - break; - default: - break; - } + timer_ce_enabled = 0; smp_mb(); + return 0; +} + +static int timer_ce_set_periodic(struct clock_event_device *evt) +{ + timer_ce_enabled = 1; + smp_mb(); + return 0; } static __init void setup_timer_ce(void) @@ -133,7 +124,9 @@ static __init void setup_timer_ce(void) ce->name = "timer_ce"; ce->rating = 100; ce->features = CLOCK_EVT_FEAT_PERIODIC; - ce->set_mode = timer_ce_set_mode; + ce->set_state_shutdown = timer_ce_shutdown; + ce->set_state_periodic = timer_ce_set_periodic; + ce->tick_resume = timer_ce_set_periodic; ce->cpumask = cpu_possible_mask; ce->shift = 32; ce->mult = div_sc(sparc_config.clock_rate, NSEC_PER_SEC, @@ -143,9 +136,9 @@ static __init void setup_timer_ce(void) static unsigned int sbus_cycles_offset(void) { - unsigned int val, offset; + u32 val, offset; - val = *master_l10_counter; + val = sbus_readl(master_l10_counter); offset = (val >> TIMER_VALUE_SHIFT) & TIMER_VALUE_MASK; /* Limit hit? */ @@ -155,7 +148,7 @@ static unsigned int sbus_cycles_offset(void) return offset; } -static cycle_t timer_cs_read(struct clocksource *cs) +static u64 timer_cs_read(struct clocksource *cs) { unsigned int seq, offset; u64 cycles; @@ -179,44 +172,36 @@ static struct clocksource timer_cs = { .rating = 100, .read = timer_cs_read, .mask = CLOCKSOURCE_MASK(64), - .shift = 2, .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; static __init int setup_timer_cs(void) { timer_cs_enabled = 1; - timer_cs.mult = clocksource_hz2mult(sparc_config.clock_rate, - timer_cs.shift); - - return clocksource_register(&timer_cs); + return clocksource_register_hz(&timer_cs, sparc_config.clock_rate); } #ifdef CONFIG_SMP -static void percpu_ce_setup(enum clock_event_mode mode, - struct clock_event_device *evt) +static int percpu_ce_shutdown(struct clock_event_device *evt) { - int cpu = __first_cpu(evt->cpumask); - - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - sparc_config.load_profile_irq(cpu, - SBUS_CLOCK_RATE / HZ); - break; - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - sparc_config.load_profile_irq(cpu, 0); - break; - default: - break; - } + int cpu = cpumask_first(evt->cpumask); + + sparc_config.load_profile_irq(cpu, 0); + return 0; +} + +static int percpu_ce_set_periodic(struct clock_event_device *evt) +{ + int cpu = cpumask_first(evt->cpumask); + + sparc_config.load_profile_irq(cpu, SBUS_CLOCK_RATE / HZ); + return 0; } static int percpu_ce_set_next_event(unsigned long delta, struct clock_event_device *evt) { - int cpu = __first_cpu(evt->cpumask); + int cpu = cpumask_first(evt->cpumask); unsigned int next = (unsigned int)delta; sparc_config.load_profile_irq(cpu, next); @@ -234,14 +219,18 @@ void register_percpu_ce(int cpu) ce->name = "percpu_ce"; ce->rating = 200; ce->features = features; - ce->set_mode = percpu_ce_setup; + ce->set_state_shutdown = percpu_ce_shutdown; + ce->set_state_periodic = percpu_ce_set_periodic; + ce->set_state_oneshot = percpu_ce_shutdown; ce->set_next_event = percpu_ce_set_next_event; ce->cpumask = cpumask_of(cpu); ce->shift = 32; ce->mult = div_sc(sparc_config.clock_rate, NSEC_PER_SEC, ce->shift); ce->max_delta_ns = clockevent_delta2ns(sparc_config.clock_rate, ce); + ce->max_delta_ticks = (unsigned long)sparc_config.clock_rate; ce->min_delta_ns = clockevent_delta2ns(100, ce); + ce->min_delta_ticks = 100; clockevents_register_device(ce); } @@ -266,6 +255,7 @@ static void mostek_write_byte(struct device *dev, u32 ofs, u8 val) static struct m48t59_plat_data m48t59_data = { .read_byte = mostek_read_byte, .write_byte = mostek_write_byte, + .yy_offset = 68, }; /* resource is set at runtime */ @@ -287,7 +277,7 @@ static int clock_probe(struct platform_device *op) return -ENODEV; /* Only the primary RTC has an address property */ - if (!of_find_property(dp, "address", NULL)) + if (!of_property_present(dp, "address")) return -ENODEV; m48t59_rtc.resource = &op->resource[0]; @@ -309,7 +299,7 @@ static int clock_probe(struct platform_device *op) return 0; } -static struct of_device_id clock_match[] = { +static const struct of_device_id clock_match[] = { { .name = "eeprom", }, @@ -320,7 +310,6 @@ static struct platform_driver clock_driver = { .probe = clock_probe, .driver = { .name = "rtc", - .owner = THIS_MODULE, .of_match_table = clock_match, }, }; @@ -365,16 +354,3 @@ void __init time_init(void) sbus_time_init(); } - -static int set_rtc_mmss(unsigned long secs) -{ - struct rtc_device *rtc = rtc_class_open("rtc0"); - int err = -1; - - if (rtc) { - err = rtc_set_mmss(rtc, secs); - rtc_class_close(rtc); - } - - return err; -} diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index c3d82b5f54ca..b32f27f929d1 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* time.c: UltraSparc timer and TOD clock support. * * Copyright (C) 1997, 2008 David S. Miller (davem@davemloft.net) @@ -27,14 +28,12 @@ #include <linux/jiffies.h> #include <linux/cpufreq.h> #include <linux/percpu.h> -#include <linux/miscdevice.h> -#include <linux/rtc.h> #include <linux/rtc/m48t59.h> #include <linux/kernel_stat.h> #include <linux/clockchips.h> #include <linux/clocksource.h> -#include <linux/of_device.h> #include <linux/platform_device.h> +#include <linux/sched/clock.h> #include <linux/ftrace.h> #include <asm/oplib.h> @@ -46,16 +45,15 @@ #include <asm/smp.h> #include <asm/sections.h> #include <asm/cpudata.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/irq_regs.h> +#include <asm/cacheflush.h> #include "entry.h" +#include "kernel.h" DEFINE_SPINLOCK(rtc_lock); -#define TICK_PRIV_BIT (1UL << 63) -#define TICKCMP_IRQ_BIT (1UL << 63) - #ifdef CONFIG_SMP unsigned long profile_pc(struct pt_regs *regs) { @@ -165,13 +163,44 @@ static unsigned long tick_add_tick(unsigned long adj) return new_tick; } -static struct sparc64_tick_ops tick_operations __read_mostly = { +/* Searches for cpu clock frequency with given cpuid in OpenBoot tree */ +static unsigned long cpuid_to_freq(phandle node, int cpuid) +{ + bool is_cpu_node = false; + unsigned long freq = 0; + char type[128]; + + if (!node) + return freq; + + if (prom_getproperty(node, "device_type", type, sizeof(type)) != -1) + is_cpu_node = (strcmp(type, "cpu") == 0); + + /* try upa-portid then cpuid to get cpuid, see prom_64.c */ + if (is_cpu_node && (prom_getint(node, "upa-portid") == cpuid || + prom_getint(node, "cpuid") == cpuid)) + freq = prom_getintdefault(node, "clock-frequency", 0); + if (!freq) + freq = cpuid_to_freq(prom_getchild(node), cpuid); + if (!freq) + freq = cpuid_to_freq(prom_getsibling(node), cpuid); + + return freq; +} + +static unsigned long tick_get_frequency(void) +{ + return cpuid_to_freq(prom_root_node, hard_smp_processor_id()); +} + +static struct sparc64_tick_ops tick_operations __cacheline_aligned = { .name = "tick", .init_tick = tick_init_tick, .disable_irq = tick_disable_irq, .get_tick = tick_get_tick, .add_tick = tick_add_tick, .add_compare = tick_add_compare, + .get_frequency = tick_get_frequency, .softint_mask = 1UL << 0, }; @@ -251,6 +280,11 @@ static int stick_add_compare(unsigned long adj) return ((long)(new_tick - (orig_tick+adj))) > 0L; } +static unsigned long stick_get_frequency(void) +{ + return prom_getintdefault(prom_root_node, "stick-frequency", 0); +} + static struct sparc64_tick_ops stick_operations __read_mostly = { .name = "stick", .init_tick = stick_init_tick, @@ -258,6 +292,7 @@ static struct sparc64_tick_ops stick_operations __read_mostly = { .get_tick = stick_get_tick, .add_tick = stick_add_tick, .add_compare = stick_add_compare, + .get_frequency = stick_get_frequency, .softint_mask = 1UL << 16, }; @@ -278,9 +313,6 @@ static struct sparc64_tick_ops stick_operations __read_mostly = { * 2) write high * 3) write low */ -#define HBIRD_STICKCMP_ADDR 0x1fe0000f060UL -#define HBIRD_STICK_ADDR 0x1fe0000f070UL - static unsigned long __hbird_read_stick(void) { unsigned long ret, tmp1, tmp2, tmp3; @@ -382,6 +414,11 @@ static int hbtick_add_compare(unsigned long adj) return ((long)(val2 - val)) > 0L; } +static unsigned long hbtick_get_frequency(void) +{ + return prom_getintdefault(prom_root_node, "stick-frequency", 0); +} + static struct sparc64_tick_ops hbtick_operations __read_mostly = { .name = "hbtick", .init_tick = hbtick_init_tick, @@ -389,24 +426,10 @@ static struct sparc64_tick_ops hbtick_operations __read_mostly = { .get_tick = hbtick_get_tick, .add_tick = hbtick_add_tick, .add_compare = hbtick_add_compare, + .get_frequency = hbtick_get_frequency, .softint_mask = 1UL << 0, }; -static unsigned long timer_ticks_per_nsec_quotient __read_mostly; - -int update_persistent_clock(struct timespec now) -{ - struct rtc_device *rtc = rtc_class_open("rtc0"); - int err = -1; - - if (rtc) { - err = rtc_set_mmss(rtc, now.tv_sec); - rtc_class_close(rtc); - } - - return err; -} - unsigned long cmos_regs; EXPORT_SYMBOL(cmos_regs); @@ -423,8 +446,8 @@ static int rtc_probe(struct platform_device *op) { struct resource *r; - printk(KERN_INFO "%s: RTC regs at 0x%llx\n", - op->dev.of_node->full_name, op->resource[0].start); + printk(KERN_INFO "%pOF: RTC regs at 0x%llx\n", + op->dev.of_node, op->resource[0].start); /* The CMOS RTC driver only accepts IORESOURCE_IO, so cons * up a fake resource so that the probe works for all cases. @@ -466,7 +489,6 @@ static struct platform_driver rtc_driver = { .probe = rtc_probe, .driver = { .name = "rtc", - .owner = THIS_MODULE, .of_match_table = rtc_match, }, }; @@ -480,8 +502,8 @@ static struct platform_device rtc_bq4802_device = { static int bq4802_probe(struct platform_device *op) { - printk(KERN_INFO "%s: BQ4802 regs at 0x%llx\n", - op->dev.of_node->full_name, op->resource[0].start); + printk(KERN_INFO "%pOF: BQ4802 regs at 0x%llx\n", + op->dev.of_node, op->resource[0].start); rtc_bq4802_device.resource = &op->resource[0]; return platform_device_register(&rtc_bq4802_device); @@ -499,7 +521,6 @@ static struct platform_driver bq4802_driver = { .probe = bq4802_probe, .driver = { .name = "bq4802", - .owner = THIS_MODULE, .of_match_table = bq4802_match, }, }; @@ -523,6 +544,7 @@ static void mostek_write_byte(struct device *dev, u32 ofs, u8 val) static struct m48t59_plat_data m48t59_data = { .read_byte = mostek_read_byte, .write_byte = mostek_write_byte, + .yy_offset = 68, }; static struct platform_device m48t59_rtc = { @@ -541,12 +563,12 @@ static int mostek_probe(struct platform_device *op) /* On an Enterprise system there can be multiple mostek clocks. * We should only match the one that is on the central FHC bus. */ - if (!strcmp(dp->parent->name, "fhc") && - strcmp(dp->parent->parent->name, "central") != 0) + if (of_node_name_eq(dp->parent, "fhc") && + !of_node_name_eq(dp->parent->parent, "central")) return -ENODEV; - printk(KERN_INFO "%s: Mostek regs at 0x%llx\n", - dp->full_name, op->resource[0].start); + printk(KERN_INFO "%pOF: Mostek regs at 0x%llx\n", + dp, op->resource[0].start); m48t59_rtc.resource = &op->resource[0]; return platform_device_register(&m48t59_rtc); @@ -563,7 +585,6 @@ static struct platform_driver mostek_driver = { .probe = mostek_probe, .driver = { .name = "mostek", - .owner = THIS_MODULE, .of_match_table = mostek_match, }, }; @@ -599,34 +620,17 @@ static int __init clock_init(void) */ fs_initcall(clock_init); -/* This is gets the master TICK_INT timer going. */ -static unsigned long sparc64_init_timers(void) +/* Return true if this is Hummingbird, aka Ultra-IIe */ +static bool is_hummingbird(void) { - struct device_node *dp; - unsigned long freq; + unsigned long ver, manuf, impl; - dp = of_find_node_by_path("/"); - if (tlb_type == spitfire) { - unsigned long ver, manuf, impl; - - __asm__ __volatile__ ("rdpr %%ver, %0" - : "=&r" (ver)); - manuf = ((ver >> 48) & 0xffff); - impl = ((ver >> 32) & 0xffff); - if (manuf == 0x17 && impl == 0x13) { - /* Hummingbird, aka Ultra-IIe */ - tick_ops = &hbtick_operations; - freq = of_getintprop_default(dp, "stick-frequency", 0); - } else { - tick_ops = &tick_operations; - freq = local_cpu_data().clock_tick; - } - } else { - tick_ops = &stick_operations; - freq = of_getintprop_default(dp, "stick-frequency", 0); - } + __asm__ __volatile__ ("rdpr %%ver, %0" + : "=&r" (ver)); + manuf = ((ver >> 48) & 0xffff); + impl = ((ver >> 32) & 0xffff); - return freq; + return (manuf == 0x17 && impl == 0x13); } struct freq_table { @@ -651,20 +655,23 @@ static int sparc64_cpufreq_notifier(struct notifier_block *nb, unsigned long val void *data) { struct cpufreq_freqs *freq = data; - unsigned int cpu = freq->cpu; - struct freq_table *ft = &per_cpu(sparc64_freq_table, cpu); + unsigned int cpu; + struct freq_table *ft; - if (!ft->ref_freq) { - ft->ref_freq = freq->old; - ft->clock_tick_ref = cpu_data(cpu).clock_tick; - } - if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || - (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) || - (val == CPUFREQ_RESUMECHANGE)) { - cpu_data(cpu).clock_tick = - cpufreq_scale(ft->clock_tick_ref, - ft->ref_freq, - freq->new); + for_each_cpu(cpu, freq->policy->cpus) { + ft = &per_cpu(sparc64_freq_table, cpu); + + if (!ft->ref_freq) { + ft->ref_freq = freq->old; + ft->clock_tick_ref = cpu_data(cpu).clock_tick; + } + + if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || + (val == CPUFREQ_POSTCHANGE && freq->old > freq->new)) { + cpu_data(cpu).clock_tick = + cpufreq_scale(ft->clock_tick_ref, ft->ref_freq, + freq->new); + } } return 0; @@ -689,42 +696,29 @@ core_initcall(register_sparc64_cpufreq_notifier); static int sparc64_next_event(unsigned long delta, struct clock_event_device *evt) { - return tick_ops->add_compare(delta) ? -ETIME : 0; + return tick_operations.add_compare(delta) ? -ETIME : 0; } -static void sparc64_timer_setup(enum clock_event_mode mode, - struct clock_event_device *evt) +static int sparc64_timer_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_RESUME: - break; - - case CLOCK_EVT_MODE_SHUTDOWN: - tick_ops->disable_irq(); - break; - - case CLOCK_EVT_MODE_PERIODIC: - case CLOCK_EVT_MODE_UNUSED: - WARN_ON(1); - break; - } + tick_operations.disable_irq(); + return 0; } static struct clock_event_device sparc64_clockevent = { - .features = CLOCK_EVT_FEAT_ONESHOT, - .set_mode = sparc64_timer_setup, - .set_next_event = sparc64_next_event, - .rating = 100, - .shift = 30, - .irq = -1, + .features = CLOCK_EVT_FEAT_ONESHOT, + .set_state_shutdown = sparc64_timer_shutdown, + .set_next_event = sparc64_next_event, + .rating = 100, + .shift = 30, + .irq = -1, }; static DEFINE_PER_CPU(struct clock_event_device, sparc64_events); void __irq_entry timer_interrupt(int irq, struct pt_regs *regs) { struct pt_regs *old_regs = set_irq_regs(regs); - unsigned long tick_mask = tick_ops->softint_mask; + unsigned long tick_mask = tick_operations.softint_mask; int cpu = smp_processor_id(); struct clock_event_device *evt = &per_cpu(sparc64_events, cpu); @@ -733,7 +727,7 @@ void __irq_entry timer_interrupt(int irq, struct pt_regs *regs) irq_enter(); local_cpu_data().irq0_irqs++; - kstat_incr_irqs_this_cpu(0, irq_to_desc(0)); + kstat_incr_irq_this_cpu(0); if (unlikely(!evt->event_handler)) { printk(KERN_WARNING @@ -759,14 +753,14 @@ void setup_sparc64_timer(void) : "=r" (pstate) : "i" (PSTATE_IE)); - tick_ops->init_tick(); + tick_operations.init_tick(); /* Restore PSTATE_IE. */ __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : /* no outputs */ : "r" (pstate)); - sevt = &__get_cpu_var(sparc64_events); + sevt = this_cpu_ptr(&sparc64_events); memcpy(sevt, &sparc64_clockevent, sizeof(*sevt)); sevt->cpumask = cpumask_of(smp_processor_id()); @@ -786,12 +780,10 @@ static unsigned long tb_ticks_per_usec __read_mostly; void __delay(unsigned long loops) { - unsigned long bclock, now; + unsigned long bclock = get_tick(); - bclock = tick_ops->get_tick(); - do { - now = tick_ops->get_tick(); - } while ((now-bclock) < loops); + while ((get_tick() - bclock) < loops) + ; } EXPORT_SYMBOL(__delay); @@ -801,34 +793,85 @@ void udelay(unsigned long usecs) } EXPORT_SYMBOL(udelay); -static cycle_t clocksource_tick_read(struct clocksource *cs) +static u64 clocksource_tick_read(struct clocksource *cs) +{ + return get_tick(); +} + +static void __init get_tick_patch(void) +{ + unsigned int *addr, *instr, i; + struct get_tick_patch *p; + + if (tlb_type == spitfire && is_hummingbird()) + return; + + for (p = &__get_tick_patch; p < &__get_tick_patch_end; p++) { + instr = (tlb_type == spitfire) ? p->tick : p->stick; + addr = (unsigned int *)(unsigned long)p->addr; + for (i = 0; i < GET_TICK_NINSTR; i++) { + addr[i] = instr[i]; + /* ensure that address is modified before flush */ + wmb(); + flushi(&addr[i]); + } + } +} + +static void __init init_tick_ops(struct sparc64_tick_ops *ops) { - return tick_ops->get_tick(); + unsigned long freq, quotient, tick; + + freq = ops->get_frequency(); + quotient = clocksource_hz2mult(freq, SPARC64_NSEC_PER_CYC_SHIFT); + tick = ops->get_tick(); + + ops->offset = (tick * quotient) >> SPARC64_NSEC_PER_CYC_SHIFT; + ops->ticks_per_nsec_quotient = quotient; + ops->frequency = freq; + tick_operations = *ops; + get_tick_patch(); +} + +void __init time_init_early(void) +{ + if (tlb_type == spitfire) { + if (is_hummingbird()) { + init_tick_ops(&hbtick_operations); + clocksource_tick.archdata.vclock_mode = VCLOCK_NONE; + } else { + init_tick_ops(&tick_operations); + clocksource_tick.archdata.vclock_mode = VCLOCK_TICK; + } + } else { + init_tick_ops(&stick_operations); + clocksource_tick.archdata.vclock_mode = VCLOCK_STICK; + } } void __init time_init(void) { - unsigned long freq = sparc64_init_timers(); + unsigned long freq; + freq = tick_operations.frequency; tb_ticks_per_usec = freq / USEC_PER_SEC; - timer_ticks_per_nsec_quotient = - clocksource_hz2mult(freq, SPARC64_NSEC_PER_CYC_SHIFT); - - clocksource_tick.name = tick_ops->name; + clocksource_tick.name = tick_operations.name; clocksource_tick.read = clocksource_tick_read; clocksource_register_hz(&clocksource_tick, freq); printk("clocksource: mult[%x] shift[%d]\n", clocksource_tick.mult, clocksource_tick.shift); - sparc64_clockevent.name = tick_ops->name; + sparc64_clockevent.name = tick_operations.name; clockevents_calc_mult_shift(&sparc64_clockevent, freq, 4); sparc64_clockevent.max_delta_ns = clockevent_delta2ns(0x7fffffffffffffffUL, &sparc64_clockevent); + sparc64_clockevent.max_delta_ticks = 0x7fffffffffffffffUL; sparc64_clockevent.min_delta_ns = clockevent_delta2ns(0xF, &sparc64_clockevent); + sparc64_clockevent.min_delta_ticks = 0xF; printk("clockevent: mult[%x] shift[%d]\n", sparc64_clockevent.mult, sparc64_clockevent.shift); @@ -838,14 +881,21 @@ void __init time_init(void) unsigned long long sched_clock(void) { - unsigned long ticks = tick_ops->get_tick(); + unsigned long quotient = tick_operations.ticks_per_nsec_quotient; + unsigned long offset = tick_operations.offset; + + /* Use barrier so the compiler emits the loads first and overlaps load + * latency with reading tick, because reading %tick/%stick is a + * post-sync instruction that will flush and restart subsequent + * instructions after it commits. + */ + barrier(); - return (ticks * timer_ticks_per_nsec_quotient) - >> SPARC64_NSEC_PER_CYC_SHIFT; + return ((get_tick() * quotient) >> SPARC64_NSEC_PER_CYC_SHIFT) - offset; } int read_current_timer(unsigned long *timer_val) { - *timer_val = tick_ops->get_tick(); + *timer_val = get_tick(); return 0; } diff --git a/arch/sparc/kernel/trampoline_32.S b/arch/sparc/kernel/trampoline_32.S index 6cdb08cdabf0..82fafeeb3a62 100644 --- a/arch/sparc/kernel/trampoline_32.S +++ b/arch/sparc/kernel/trampoline_32.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * trampoline.S: SMP cpu boot-up trampoline code. * @@ -5,7 +6,6 @@ * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ -#include <linux/init.h> #include <asm/head.h> #include <asm/psr.h> #include <asm/page.h> @@ -18,7 +18,6 @@ .globl sun4m_cpu_startup .globl sun4d_cpu_startup - __CPUINIT .align 4 /* When we start up a cpu for the first time it enters this routine. @@ -94,7 +93,6 @@ smp_panic: /* CPUID in bootbus can be found at PA 0xff0140000 */ #define SUN4D_BOOTBUS_CPUID 0xf0140000 - __CPUINIT .align 4 sun4d_cpu_startup: @@ -146,7 +144,6 @@ sun4d_cpu_startup: b,a smp_panic - __CPUINIT .align 4 .global leon_smp_cpu_startup, smp_penguin_ctable diff --git a/arch/sparc/kernel/trampoline_64.S b/arch/sparc/kernel/trampoline_64.S index 2e973a26fbda..51bf1eb92a36 100644 --- a/arch/sparc/kernel/trampoline_64.S +++ b/arch/sparc/kernel/trampoline_64.S @@ -1,11 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * trampoline.S: Jump start slave processors on sparc64. * * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) */ -#include <linux/init.h> +#include <linux/pgtable.h> #include <asm/head.h> #include <asm/asi.h> #include <asm/lsu.h> @@ -13,7 +14,6 @@ #include <asm/dcu.h> #include <asm/pstate.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/spitfire.h> #include <asm/processor.h> #include <asm/thread_info.h> @@ -32,13 +32,11 @@ itlb_load: dtlb_load: .asciz "SUNW,dtlb-load" - /* XXX __cpuinit this thing XXX */ #define TRAMP_STACK_SIZE 1024 .align 16 tramp_stack: .skip TRAMP_STACK_SIZE - __CPUINIT .align 8 .globl sparc64_cpu_startup, sparc64_cpu_startup_end sparc64_cpu_startup: @@ -112,10 +110,13 @@ startup_continue: brnz,pn %g1, 1b nop - sethi %hi(p1275buf), %g2 - or %g2, %lo(p1275buf), %g2 - ldx [%g2 + 0x10], %l2 - add %l2, -(192 + 128), %sp + /* Get onto temporary stack which will be in the locked + * kernel image. + */ + sethi %hi(tramp_stack), %g1 + or %g1, %lo(tramp_stack), %g1 + add %g1, TRAMP_STACK_SIZE, %g1 + sub %g1, STACKFRAME_SZ + STACK_BIAS + 256, %sp flushw /* Setup the loop variables: @@ -131,7 +132,6 @@ startup_continue: clr %l5 sethi %hi(num_kernel_image_mappings), %l6 lduw [%l6 + %lo(num_kernel_image_mappings)], %l6 - add %l6, 1, %l6 mov 15, %l7 BRANCH_IF_ANY_CHEETAH(g1,g5,2f) @@ -224,7 +224,6 @@ niagara_lock_tlb: clr %l5 sethi %hi(num_kernel_image_mappings), %l6 lduw [%l6 + %lo(num_kernel_image_mappings)], %l6 - add %l6, 1, %l6 1: mov HV_FAST_MMU_MAP_PERM_ADDR, %o5 @@ -399,7 +398,6 @@ after_lock_tlb: sllx %g5, THREAD_SHIFT, %g5 sub %g5, (STACKFRAME_SZ + STACK_BIAS), %g5 add %g6, %g5, %sp - mov 0, %fp rdpr %pstate, %o1 or %o1, PSTATE_IE, %o1 diff --git a/arch/sparc/kernel/traps_32.c b/arch/sparc/kernel/traps_32.c index 662982946a89..bb149f6cc34b 100644 --- a/arch/sparc/kernel/traps_32.c +++ b/arch/sparc/kernel/traps_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * arch/sparc/kernel/traps.c * @@ -9,18 +10,21 @@ * I hate traps on the sparc, grrr... */ -#include <linux/sched.h> /* for jiffies */ +#include <linux/cpu.h> +#include <linux/sched/mm.h> +#include <linux/sched/debug.h> +#include <linux/mm_types.h> #include <linux/kernel.h> #include <linux/signal.h> #include <linux/smp.h> #include <linux/kdebug.h> #include <linux/export.h> +#include <linux/pgtable.h> #include <asm/delay.h> #include <asm/ptrace.h> #include <asm/oplib.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/unistd.h> #include <asm/traps.h> @@ -44,7 +48,7 @@ static void instruction_dump(unsigned long *pc) #define __SAVE __asm__ __volatile__("save %sp, -0x40, %sp\n\t") #define __RESTORE __asm__ __volatile__("restore %g0, %g0, %g0\n\t") -void die_if_kernel(char *str, struct pt_regs *regs) +void __noreturn die_if_kernel(char *str, struct pt_regs *regs) { static int die_counter; int count = 0; @@ -83,15 +87,11 @@ void die_if_kernel(char *str, struct pt_regs *regs) } printk("Instruction DUMP:"); instruction_dump ((unsigned long *) regs->pc); - if(regs->psr & PSR_PS) - do_exit(SIGKILL); - do_exit(SIGSEGV); + make_task_dead((regs->psr & PSR_PS) ? SIGKILL : SIGSEGV); } void do_hw_interrupt(struct pt_regs *regs, unsigned long type) { - siginfo_t info; - if(type < 0x80) { /* Sun OS's puke from bad traps, Linux survives! */ printk("Unimplemented Sparc TRAP, type = %02lx\n", type); @@ -101,19 +101,13 @@ void do_hw_interrupt(struct pt_regs *regs, unsigned long type) if(regs->psr & PSR_PS) die_if_kernel("Kernel bad trap", regs); - info.si_signo = SIGILL; - info.si_errno = 0; - info.si_code = ILL_ILLTRP; - info.si_addr = (void __user *)regs->pc; - info.si_trapno = type - 0x80; - force_sig_info(SIGILL, &info, current); + force_sig_fault_trapno(SIGILL, ILL_ILLTRP, + (void __user *)regs->pc, type - 0x80); } void do_illegal_instruction(struct pt_regs *regs, unsigned long pc, unsigned long npc, unsigned long psr) { - siginfo_t info; - if(psr & PSR_PS) die_if_kernel("Kernel illegal instruction", regs); #ifdef TRAP_DEBUG @@ -121,27 +115,15 @@ void do_illegal_instruction(struct pt_regs *regs, unsigned long pc, unsigned lon regs->pc, *(unsigned long *)regs->pc); #endif - info.si_signo = SIGILL; - info.si_errno = 0; - info.si_code = ILL_ILLOPC; - info.si_addr = (void __user *)pc; - info.si_trapno = 0; - send_sig_info(SIGILL, &info, current); + send_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)pc, current); } void do_priv_instruction(struct pt_regs *regs, unsigned long pc, unsigned long npc, unsigned long psr) { - siginfo_t info; - if(psr & PSR_PS) die_if_kernel("Penguin instruction from Penguin mode??!?!", regs); - info.si_signo = SIGILL; - info.si_errno = 0; - info.si_code = ILL_PRVOPC; - info.si_addr = (void __user *)pc; - info.si_trapno = 0; - send_sig_info(SIGILL, &info, current); + send_sig_fault(SIGILL, ILL_PRVOPC, (void __user *)pc, current); } /* XXX User may want to be allowed to do this. XXX */ @@ -149,8 +131,6 @@ void do_priv_instruction(struct pt_regs *regs, unsigned long pc, unsigned long n void do_memaccess_unaligned(struct pt_regs *regs, unsigned long pc, unsigned long npc, unsigned long psr) { - siginfo_t info; - if(regs->psr & PSR_PS) { printk("KERNEL MNA at pc %08lx npc %08lx called by %08lx\n", pc, npc, regs->u_regs[UREG_RETPC]); @@ -162,12 +142,9 @@ void do_memaccess_unaligned(struct pt_regs *regs, unsigned long pc, unsigned lon instruction_dump ((unsigned long *) regs->pc); printk ("do_MNA!\n"); #endif - info.si_signo = SIGBUS; - info.si_errno = 0; - info.si_code = BUS_ADRALN; - info.si_addr = /* FIXME: Should dig out mna address */ (void *)0; - info.si_trapno = 0; - send_sig_info(SIGBUS, &info, current); + send_sig_fault(SIGBUS, BUS_ADRALN, + /* FIXME: Should dig out mna address */ (void *)0, + current); } static unsigned long init_fsr = 0x0UL; @@ -219,15 +196,13 @@ static unsigned long fake_fsr; static unsigned long fake_queue[32] __attribute__ ((aligned (8))); static unsigned long fake_depth; -extern int do_mathemu(struct pt_regs *, struct task_struct *); - void do_fpe_trap(struct pt_regs *regs, unsigned long pc, unsigned long npc, unsigned long psr) { static int calls; - siginfo_t info; unsigned long fsr; int ret = 0; + int code; #ifndef CONFIG_SMP struct task_struct *fpt = last_task_used_math; #else @@ -302,24 +277,20 @@ void do_fpe_trap(struct pt_regs *regs, unsigned long pc, unsigned long npc, } fsr = fpt->thread.fsr; - info.si_signo = SIGFPE; - info.si_errno = 0; - info.si_addr = (void __user *)pc; - info.si_trapno = 0; - info.si_code = __SI_FAULT; + code = FPE_FLTUNK; if ((fsr & 0x1c000) == (1 << 14)) { if (fsr & 0x10) - info.si_code = FPE_FLTINV; + code = FPE_FLTINV; else if (fsr & 0x08) - info.si_code = FPE_FLTOVF; + code = FPE_FLTOVF; else if (fsr & 0x04) - info.si_code = FPE_FLTUND; + code = FPE_FLTUND; else if (fsr & 0x02) - info.si_code = FPE_FLTDIV; + code = FPE_FLTDIV; else if (fsr & 0x01) - info.si_code = FPE_FLTRES; + code = FPE_FLTRES; } - send_sig_info(SIGFPE, &info, fpt); + send_sig_fault(SIGFPE, code, (void __user *)pc, fpt); #ifndef CONFIG_SMP last_task_used_math = NULL; #endif @@ -331,16 +302,9 @@ void do_fpe_trap(struct pt_regs *regs, unsigned long pc, unsigned long npc, void handle_tag_overflow(struct pt_regs *regs, unsigned long pc, unsigned long npc, unsigned long psr) { - siginfo_t info; - if(psr & PSR_PS) die_if_kernel("Penguin overflow trap from kernel mode", regs); - info.si_signo = SIGEMT; - info.si_errno = 0; - info.si_code = EMT_TAGOVF; - info.si_addr = (void __user *)pc; - info.si_trapno = 0; - send_sig_info(SIGEMT, &info, current); + send_sig_fault(SIGEMT, EMT_TAGOVF, (void __user *)pc, current); } void handle_watchpoint(struct pt_regs *regs, unsigned long pc, unsigned long npc, @@ -358,61 +322,33 @@ void handle_watchpoint(struct pt_regs *regs, unsigned long pc, unsigned long npc void handle_reg_access(struct pt_regs *regs, unsigned long pc, unsigned long npc, unsigned long psr) { - siginfo_t info; - #ifdef TRAP_DEBUG printk("Register Access Exception at PC %08lx NPC %08lx PSR %08lx\n", pc, npc, psr); #endif - info.si_signo = SIGBUS; - info.si_errno = 0; - info.si_code = BUS_OBJERR; - info.si_addr = (void __user *)pc; - info.si_trapno = 0; - force_sig_info(SIGBUS, &info, current); + force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)pc); } void handle_cp_disabled(struct pt_regs *regs, unsigned long pc, unsigned long npc, unsigned long psr) { - siginfo_t info; - - info.si_signo = SIGILL; - info.si_errno = 0; - info.si_code = ILL_COPROC; - info.si_addr = (void __user *)pc; - info.si_trapno = 0; - send_sig_info(SIGILL, &info, current); + send_sig_fault(SIGILL, ILL_COPROC, (void __user *)pc, current); } void handle_cp_exception(struct pt_regs *regs, unsigned long pc, unsigned long npc, unsigned long psr) { - siginfo_t info; - #ifdef TRAP_DEBUG printk("Co-Processor Exception at PC %08lx NPC %08lx PSR %08lx\n", pc, npc, psr); #endif - info.si_signo = SIGILL; - info.si_errno = 0; - info.si_code = ILL_COPROC; - info.si_addr = (void __user *)pc; - info.si_trapno = 0; - send_sig_info(SIGILL, &info, current); + send_sig_fault(SIGILL, ILL_COPROC, (void __user *)pc, current); } void handle_hw_divzero(struct pt_regs *regs, unsigned long pc, unsigned long npc, unsigned long psr) { - siginfo_t info; - - info.si_signo = SIGFPE; - info.si_errno = 0; - info.si_code = FPE_INTDIV; - info.si_addr = (void __user *)pc; - info.si_trapno = 0; - send_sig_info(SIGFPE, &info, current); + send_sig_fault(SIGFPE, FPE_INTDIV, (void __user *)pc, current); } #ifdef CONFIG_DEBUG_BUGVERBOSE @@ -435,7 +371,6 @@ void trap_init(void) /* Force linker to barf if mismatched */ if (TI_UWINMASK != offsetof(struct thread_info, uwinmask) || TI_TASK != offsetof(struct thread_info, task) || - TI_EXECDOMAIN != offsetof(struct thread_info, exec_domain) || TI_FLAGS != offsetof(struct thread_info, flags) || TI_CPU != offsetof(struct thread_info, cpu) || TI_PREEMPT != offsetof(struct thread_info, preempt_count) || @@ -451,7 +386,7 @@ void trap_init(void) thread_info_offsets_are_bolixed_pete(); /* Attach to the address space of init_task. */ - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; /* NOTE: Other cpus have this done as they are started diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index b3f833ab90eb..28cb0d66ab40 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* arch/sparc64/kernel/traps.c * * Copyright (C) 1995,1997,2008,2009,2012 David S. Miller (davem@davemloft.net) @@ -8,27 +9,30 @@ * I like traps on v9, :)))) */ -#include <linux/module.h> -#include <linux/sched.h> +#include <linux/cpu.h> +#include <linux/extable.h> +#include <linux/sched/mm.h> +#include <linux/sched/debug.h> #include <linux/linkage.h> #include <linux/kernel.h> #include <linux/signal.h> #include <linux/smp.h> #include <linux/mm.h> #include <linux/init.h> +#include <linux/kallsyms.h> #include <linux/kdebug.h> #include <linux/ftrace.h> #include <linux/reboot.h> #include <linux/gfp.h> +#include <linux/context_tracking.h> #include <asm/smp.h> #include <asm/delay.h> #include <asm/ptrace.h> #include <asm/oplib.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/unistd.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/fpumacro.h> #include <asm/lsu.h> #include <asm/dcu.h> @@ -42,8 +46,10 @@ #include <asm/prom.h> #include <asm/memctrl.h> #include <asm/cacheflush.h> +#include <asm/setup.h> #include "entry.h" +#include "kernel.h" #include "kstack.h" /* When an irrecoverable trap occurs at tl > 0, the trap entry @@ -82,8 +88,7 @@ static void dump_tl1_traplog(struct tl1_traplog *p) void bad_trap(struct pt_regs *regs, long lvl) { - char buffer[32]; - siginfo_t info; + char buffer[36]; if (notify_die(DIE_TRAP, "bad trap", regs, 0, lvl, SIGTRAP) == NOTIFY_STOP) @@ -103,17 +108,13 @@ void bad_trap(struct pt_regs *regs, long lvl) regs->tpc &= 0xffffffff; regs->tnpc &= 0xffffffff; } - info.si_signo = SIGILL; - info.si_errno = 0; - info.si_code = ILL_ILLTRP; - info.si_addr = (void __user *)regs->tpc; - info.si_trapno = lvl; - force_sig_info(SIGILL, &info, current); + force_sig_fault_trapno(SIGILL, ILL_ILLTRP, + (void __user *)regs->tpc, lvl); } void bad_trap_tl1(struct pt_regs *regs, long lvl) { - char buffer[32]; + char buffer[36]; if (notify_die(DIE_TRAP_TL1, "bad trap tl1", regs, 0, lvl, SIGTRAP) == NOTIFY_STOP) @@ -186,11 +187,11 @@ EXPORT_SYMBOL_GPL(unregister_dimm_printer); void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) { - siginfo_t info; + enum ctx_state prev_state = exception_enter(); if (notify_die(DIE_TRAP, "instruction access exception", regs, 0, 0x8, SIGTRAP) == NOTIFY_STOP) - return; + goto out; if (regs->tstate & TSTATE_PRIV) { printk("spitfire_insn_access_exception: SFSR[%016lx] " @@ -201,12 +202,9 @@ void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, un regs->tpc &= 0xffffffff; regs->tnpc &= 0xffffffff; } - info.si_signo = SIGSEGV; - info.si_errno = 0; - info.si_code = SEGV_MAPERR; - info.si_addr = (void __user *)regs->tpc; - info.si_trapno = 0; - force_sig_info(SIGSEGV, &info, current); + force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)regs->tpc); +out: + exception_exit(prev_state); } void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) @@ -223,7 +221,6 @@ void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsig { unsigned short type = (type_ctx >> 16); unsigned short ctx = (type_ctx & 0xffff); - siginfo_t info; if (notify_die(DIE_TRAP, "instruction access exception", regs, 0, 0x8, SIGTRAP) == NOTIFY_STOP) @@ -240,12 +237,7 @@ void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsig regs->tpc &= 0xffffffff; regs->tnpc &= 0xffffffff; } - info.si_signo = SIGSEGV; - info.si_errno = 0; - info.si_code = SEGV_MAPERR; - info.si_addr = (void __user *) addr; - info.si_trapno = 0; - force_sig_info(SIGSEGV, &info, current); + force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *) addr); } void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) @@ -258,13 +250,51 @@ void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, u sun4v_insn_access_exception(regs, addr, type_ctx); } +static bool is_no_fault_exception(struct pt_regs *regs) +{ + unsigned char asi; + u32 insn; + + if (get_user(insn, (u32 __user *)regs->tpc) == -EFAULT) + return false; + + /* + * Must do a little instruction decoding here in order to + * decide on a course of action. The bits of interest are: + * insn[31:30] = op, where 3 indicates the load/store group + * insn[24:19] = op3, which identifies individual opcodes + * insn[13] indicates an immediate offset + * op3[4]=1 identifies alternate space instructions + * op3[5:4]=3 identifies floating point instructions + * op3[2]=1 identifies stores + * See "Opcode Maps" in the appendix of any Sparc V9 + * architecture spec for full details. + */ + if ((insn & 0xc0800000) == 0xc0800000) { /* op=3, op3[4]=1 */ + if (insn & 0x2000) /* immediate offset */ + asi = (regs->tstate >> 24); /* saved %asi */ + else + asi = (insn >> 5); /* immediate asi */ + if ((asi & 0xf6) == ASI_PNF) { + if (insn & 0x200000) /* op3[2], stores */ + return false; + if (insn & 0x1000000) /* op3[5:4]=3 (fp) */ + handle_ldf_stq(insn, regs); + else + handle_ld_nf(insn, regs); + return true; + } + } + return false; +} + void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) { - siginfo_t info; + enum ctx_state prev_state = exception_enter(); if (notify_die(DIE_TRAP, "data access exception", regs, 0, 0x30, SIGTRAP) == NOTIFY_STOP) - return; + goto out; if (regs->tstate & TSTATE_PRIV) { /* Test if this comes from uaccess places. */ @@ -280,7 +310,7 @@ void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, un #endif regs->tpc = entry->fixup; regs->tnpc = regs->tpc + 4; - return; + goto out; } /* Shit... */ printk("spitfire_data_access_exception: SFSR[%016lx] " @@ -288,12 +318,12 @@ void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, un die_if_kernel("Dax", regs); } - info.si_signo = SIGSEGV; - info.si_errno = 0; - info.si_code = SEGV_MAPERR; - info.si_addr = (void __user *)sfar; - info.si_trapno = 0; - force_sig_info(SIGSEGV, &info, current); + if (is_no_fault_exception(regs)) + return; + + force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)sfar); +out: + exception_exit(prev_state); } void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) @@ -310,7 +340,6 @@ void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsig { unsigned short type = (type_ctx >> 16); unsigned short ctx = (type_ctx & 0xffff); - siginfo_t info; if (notify_die(DIE_TRAP, "data access exception", regs, 0, 0x8, SIGTRAP) == NOTIFY_STOP) @@ -342,12 +371,29 @@ void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsig regs->tpc &= 0xffffffff; regs->tnpc &= 0xffffffff; } - info.si_signo = SIGSEGV; - info.si_errno = 0; - info.si_code = SEGV_MAPERR; - info.si_addr = (void __user *) addr; - info.si_trapno = 0; - force_sig_info(SIGSEGV, &info, current); + if (is_no_fault_exception(regs)) + return; + + /* MCD (Memory Corruption Detection) disabled trap (TT=0x19) in HV + * is vectored thorugh data access exception trap with fault type + * set to HV_FAULT_TYPE_MCD_DIS. Check for MCD disabled trap. + * Accessing an address with invalid ASI for the address, for + * example setting an ADI tag on an address with ASI_MCD_PRIMARY + * when TTE.mcd is not set for the VA, is also vectored into + * kerbel by HV as data access exception with fault type set to + * HV_FAULT_TYPE_INV_ASI. + */ + switch (type) { + case HV_FAULT_TYPE_INV_ASI: + force_sig_fault(SIGILL, ILL_ILLADR, (void __user *)addr); + break; + case HV_FAULT_TYPE_MCD_DIS: + force_sig_fault(SIGSEGV, SEGV_ACCADI, (void __user *)addr); + break; + default: + force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)addr); + break; + } } void sun4v_data_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) @@ -488,8 +534,6 @@ static void spitfire_cee_log(unsigned long afsr, unsigned long afar, unsigned lo static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long tt, int tl1, struct pt_regs *regs) { - siginfo_t info; - printk(KERN_WARNING "CPU[%d]: Uncorrectable Error AFSR[%lx] " "AFAR[%lx] UDBL[%lx] UDBH[%ld] TT[%lx] TL>1[%d]\n", smp_processor_id(), afsr, afar, udbl, udbh, tt, tl1); @@ -524,12 +568,7 @@ static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned lon regs->tpc &= 0xffffffff; regs->tnpc &= 0xffffffff; } - info.si_signo = SIGBUS; - info.si_errno = 0; - info.si_code = BUS_OBJERR; - info.si_addr = (void *)0; - info.si_trapno = 0; - force_sig_info(SIGBUS, &info, current); + force_sig_fault(SIGBUS, BUS_OBJERR, (void *)0); } void spitfire_access_error(struct pt_regs *regs, unsigned long status_encoded, unsigned long afar) @@ -859,7 +898,7 @@ void __init cheetah_ecache_flush_init(void) /* Now allocate error trap reporting scoreboard. */ sz = NR_CPUS * (2 * sizeof(struct cheetah_err_info)); - for (order = 0; order < MAX_ORDER; order++) { + for (order = 0; order < NR_PAGE_ORDERS; order++) { if ((PAGE_SIZE << order) >= sz) break; } @@ -1792,6 +1831,7 @@ struct sun4v_error_entry { #define SUN4V_ERR_ATTRS_ASI 0x00000080 #define SUN4V_ERR_ATTRS_PRIV_REG 0x00000100 #define SUN4V_ERR_ATTRS_SPSTATE_MSK 0x00000600 +#define SUN4V_ERR_ATTRS_MCD 0x00000800 #define SUN4V_ERR_ATTRS_SPSTATE_SHFT 9 #define SUN4V_ERR_ATTRS_MODE_MSK 0x03000000 #define SUN4V_ERR_ATTRS_MODE_SHFT 24 @@ -1989,11 +2029,55 @@ static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent, } } +/* Handle memory corruption detected error which is vectored in + * through resumable error trap. + */ +static void do_mcd_err(struct pt_regs *regs, struct sun4v_error_entry ent) +{ + if (notify_die(DIE_TRAP, "MCD error", regs, 0, 0x34, + SIGSEGV) == NOTIFY_STOP) + return; + + if (regs->tstate & TSTATE_PRIV) { + /* MCD exception could happen because the task was + * running a system call with MCD enabled and passed a + * non-versioned pointer or pointer with bad version + * tag to the system call. In such cases, hypervisor + * places the address of offending instruction in the + * resumable error report. This is a deferred error, + * so the read/write that caused the trap was potentially + * retired long time back and we may have no choice + * but to send SIGSEGV to the process. + */ + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->tpc); + if (entry) { + /* Looks like a bad syscall parameter */ +#ifdef DEBUG_EXCEPTIONS + pr_emerg("Exception: PC<%016lx> faddr<UNKNOWN>\n", + regs->tpc); + pr_emerg("EX_TABLE: insn<%016lx> fixup<%016lx>\n", + ent.err_raddr, entry->fixup); +#endif + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + return; + } + } + + /* Send SIGSEGV to the userspace process with the right signal + * code + */ + force_sig_fault(SIGSEGV, SEGV_ADIDERR, (void __user *)ent.err_raddr); +} + /* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate. * Log the event and clear the first word of the entry. */ void sun4v_resum_error(struct pt_regs *regs, unsigned long offset) { + enum ctx_state prev_state = exception_enter(); struct sun4v_error_entry *ent, local_copy; struct trap_per_cpu *tb; unsigned long paddr; @@ -2022,12 +2106,22 @@ void sun4v_resum_error(struct pt_regs *regs, unsigned long offset) pr_info("Shutdown request, %u seconds...\n", local_copy.err_secs); orderly_poweroff(true); + goto out; + } + + /* If this is a memory corruption detected error vectored in + * by HV through resumable error trap, call the handler + */ + if (local_copy.err_attrs & SUN4V_ERR_ATTRS_MCD) { + do_mcd_err(regs, local_copy); return; } sun4v_log_error(regs, &local_copy, cpu, KERN_ERR "RESUMABLE ERROR", &sun4v_resum_oflow_cnt); +out: + exception_exit(prev_state); } /* If we try to printk() we'll probably make matters worse, by trying @@ -2039,6 +2133,64 @@ void sun4v_resum_overflow(struct pt_regs *regs) atomic_inc(&sun4v_resum_oflow_cnt); } +/* Given a set of registers, get the virtual addressi that was being accessed + * by the faulting instructions at tpc. + */ +static unsigned long sun4v_get_vaddr(struct pt_regs *regs) +{ + unsigned int insn; + + if (!copy_from_user(&insn, (void __user *)regs->tpc, 4)) { + return compute_effective_address(regs, insn, + (insn >> 25) & 0x1f); + } + return 0; +} + +/* Attempt to handle non-resumable errors generated from userspace. + * Returns true if the signal was handled, false otherwise. + */ +static bool sun4v_nonresum_error_user_handled(struct pt_regs *regs, + struct sun4v_error_entry *ent) +{ + unsigned int attrs = ent->err_attrs; + + if (attrs & SUN4V_ERR_ATTRS_MEMORY) { + unsigned long addr = ent->err_raddr; + + if (addr == ~(u64)0) { + /* This seems highly unlikely to ever occur */ + pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory error detected in unknown location!\n"); + } else { + unsigned long page_cnt = DIV_ROUND_UP(ent->err_size, + PAGE_SIZE); + + /* Break the unfortunate news. */ + pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory failed at %016lX\n", + addr); + pr_emerg("SUN4V NON-RECOVERABLE ERROR: Claiming %lu ages.\n", + page_cnt); + + while (page_cnt-- > 0) { + if (pfn_valid(addr >> PAGE_SHIFT)) + get_page(pfn_to_page(addr >> PAGE_SHIFT)); + addr += PAGE_SIZE; + } + } + force_sig(SIGKILL); + + return true; + } + if (attrs & SUN4V_ERR_ATTRS_PIO) { + force_sig_fault(SIGBUS, BUS_ADRERR, + (void __user *)sun4v_get_vaddr(regs)); + return true; + } + + /* Default to doing nothing */ + return false; +} + /* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate. * Log the event, clear the first word of the entry, and die. */ @@ -2063,6 +2215,12 @@ void sun4v_nonresum_error(struct pt_regs *regs, unsigned long offset) put_cpu(); + if (!(regs->tstate & TSTATE_PRIV) && + sun4v_nonresum_error_user_handled(regs, &local_copy)) { + /* DON'T PANIC: This userspace error was handled. */ + return; + } + #ifdef CONFIG_PCI /* Check for the special PCI poke sequence. */ if (pci_poke_in_progress && pci_poke_cpu == cpu) { @@ -2092,6 +2250,11 @@ void sun4v_nonresum_overflow(struct pt_regs *regs) atomic_inc(&sun4v_nonresum_oflow_cnt); } +static void sun4v_tlb_error(struct pt_regs *regs) +{ + die_if_kernel("TLB/TSB error", regs); +} + unsigned long sun4v_err_itlb_vaddr; unsigned long sun4v_err_itlb_ctx; unsigned long sun4v_err_itlb_pte; @@ -2099,8 +2262,7 @@ unsigned long sun4v_err_itlb_error; void sun4v_itlb_error_report(struct pt_regs *regs, int tl) { - if (tl > 1) - dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n", regs->tpc, tl); @@ -2113,7 +2275,7 @@ void sun4v_itlb_error_report(struct pt_regs *regs, int tl) sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx, sun4v_err_itlb_pte, sun4v_err_itlb_error); - prom_halt(); + sun4v_tlb_error(regs); } unsigned long sun4v_err_dtlb_vaddr; @@ -2123,8 +2285,7 @@ unsigned long sun4v_err_dtlb_error; void sun4v_dtlb_error_report(struct pt_regs *regs, int tl) { - if (tl > 1) - dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n", regs->tpc, tl); @@ -2137,7 +2298,7 @@ void sun4v_dtlb_error_report(struct pt_regs *regs, int tl) sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx, sun4v_err_dtlb_pte, sun4v_err_dtlb_error); - prom_halt(); + sun4v_tlb_error(regs); } void hypervisor_tlbop_error(unsigned long err, unsigned long op) @@ -2152,59 +2313,58 @@ void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op) err, op); } -void do_fpe_common(struct pt_regs *regs) +static void do_fpe_common(struct pt_regs *regs) { if (regs->tstate & TSTATE_PRIV) { regs->tpc = regs->tnpc; regs->tnpc += 4; } else { unsigned long fsr = current_thread_info()->xfsr[0]; - siginfo_t info; + int code; if (test_thread_flag(TIF_32BIT)) { regs->tpc &= 0xffffffff; regs->tnpc &= 0xffffffff; } - info.si_signo = SIGFPE; - info.si_errno = 0; - info.si_addr = (void __user *)regs->tpc; - info.si_trapno = 0; - info.si_code = __SI_FAULT; + code = FPE_FLTUNK; if ((fsr & 0x1c000) == (1 << 14)) { if (fsr & 0x10) - info.si_code = FPE_FLTINV; + code = FPE_FLTINV; else if (fsr & 0x08) - info.si_code = FPE_FLTOVF; + code = FPE_FLTOVF; else if (fsr & 0x04) - info.si_code = FPE_FLTUND; + code = FPE_FLTUND; else if (fsr & 0x02) - info.si_code = FPE_FLTDIV; + code = FPE_FLTDIV; else if (fsr & 0x01) - info.si_code = FPE_FLTRES; + code = FPE_FLTRES; } - force_sig_info(SIGFPE, &info, current); + force_sig_fault(SIGFPE, code, (void __user *)regs->tpc); } } void do_fpieee(struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); + if (notify_die(DIE_TRAP, "fpu exception ieee", regs, 0, 0x24, SIGFPE) == NOTIFY_STOP) - return; + goto out; do_fpe_common(regs); +out: + exception_exit(prev_state); } -extern int do_mathemu(struct pt_regs *, struct fpustate *, bool); - void do_fpother(struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); struct fpustate *f = FPUSTATE; int ret = 0; if (notify_die(DIE_TRAP, "fpu exception other", regs, 0, 0x25, SIGFPE) == NOTIFY_STOP) - return; + goto out; switch ((current_thread_info()->xfsr[0] & 0x1c000)) { case (2 << 14): /* unfinished_FPop */ @@ -2213,17 +2373,19 @@ void do_fpother(struct pt_regs *regs) break; } if (ret) - return; + goto out; do_fpe_common(regs); +out: + exception_exit(prev_state); } void do_tof(struct pt_regs *regs) { - siginfo_t info; + enum ctx_state prev_state = exception_enter(); if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs, 0, 0x26, SIGEMT) == NOTIFY_STOP) - return; + goto out; if (regs->tstate & TSTATE_PRIV) die_if_kernel("Penguin overflow trap from kernel mode", regs); @@ -2231,21 +2393,18 @@ void do_tof(struct pt_regs *regs) regs->tpc &= 0xffffffff; regs->tnpc &= 0xffffffff; } - info.si_signo = SIGEMT; - info.si_errno = 0; - info.si_code = EMT_TAGOVF; - info.si_addr = (void __user *)regs->tpc; - info.si_trapno = 0; - force_sig_info(SIGEMT, &info, current); + force_sig_fault(SIGEMT, EMT_TAGOVF, (void __user *)regs->tpc); +out: + exception_exit(prev_state); } void do_div0(struct pt_regs *regs) { - siginfo_t info; + enum ctx_state prev_state = exception_enter(); if (notify_die(DIE_TRAP, "integer division by zero", regs, 0, 0x28, SIGFPE) == NOTIFY_STOP) - return; + goto out; if (regs->tstate & TSTATE_PRIV) die_if_kernel("TL0: Kernel divide by zero.", regs); @@ -2253,12 +2412,9 @@ void do_div0(struct pt_regs *regs) regs->tpc &= 0xffffffff; regs->tnpc &= 0xffffffff; } - info.si_signo = SIGFPE; - info.si_errno = 0; - info.si_code = FPE_INTDIV; - info.si_addr = (void __user *)regs->tpc; - info.si_trapno = 0; - force_sig_info(SIGFPE, &info, current); + force_sig_fault(SIGFPE, FPE_INTDIV, (void __user *)regs->tpc); +out: + exception_exit(prev_state); } static void instruction_dump(unsigned int *pc) @@ -2291,7 +2447,7 @@ static void user_instruction_dump(unsigned int __user *pc) printk("\n"); } -void show_stack(struct task_struct *tsk, unsigned long *_ksp) +void show_stack(struct task_struct *tsk, unsigned long *_ksp, const char *loglvl) { unsigned long fp, ksp; struct thread_info *tp; @@ -2315,7 +2471,7 @@ void show_stack(struct task_struct *tsk, unsigned long *_ksp) fp = ksp + STACK_BIAS; - printk("Call Trace:\n"); + printk("%sCall Trace:\n", loglvl); do { struct sparc_stackf *sf; struct pt_regs *regs; @@ -2336,13 +2492,14 @@ void show_stack(struct task_struct *tsk, unsigned long *_ksp) fp = (unsigned long)sf->fp + STACK_BIAS; } - printk(" [%016lx] %pS\n", pc, (void *) pc); + print_ip_sym(loglvl, pc); #ifdef CONFIG_FUNCTION_GRAPH_TRACER if ((pc + 8UL) == (unsigned long) &return_to_handler) { - int index = tsk->curr_ret_stack; - if (tsk->ret_stack && index >= graph) { - pc = tsk->ret_stack[index - graph].ret; - printk(" [%016lx] %pS\n", pc, (void *) pc); + struct ftrace_ret_stack *ret_stack; + ret_stack = ftrace_graph_get_ret_stack(tsk, graph); + if (ret_stack) { + pc = ret_stack->ret; + print_ip_sym(loglvl, pc); graph++; } } @@ -2360,7 +2517,7 @@ static inline struct reg_window *kernel_stack_up(struct reg_window *rw) return (struct reg_window *) (fp + STACK_BIAS); } -void die_if_kernel(char *str, struct pt_regs *regs) +void __noreturn die_if_kernel(char *str, struct pt_regs *regs) { static int die_counter; int count = 0; @@ -2401,28 +2558,25 @@ void die_if_kernel(char *str, struct pt_regs *regs) } user_instruction_dump ((unsigned int __user *) regs->tpc); } - if (regs->tstate & TSTATE_PRIV) - do_exit(SIGKILL); - do_exit(SIGSEGV); + if (panic_on_oops) + panic("Fatal exception"); + make_task_dead((regs->tstate & TSTATE_PRIV)? SIGKILL : SIGSEGV); } EXPORT_SYMBOL(die_if_kernel); #define VIS_OPCODE_MASK ((0x3 << 30) | (0x3f << 19)) #define VIS_OPCODE_VAL ((0x2 << 30) | (0x36 << 19)) -extern int handle_popc(u32 insn, struct pt_regs *regs); -extern int handle_ldf_stq(u32 insn, struct pt_regs *regs); - void do_illegal_instruction(struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); unsigned long pc = regs->tpc; unsigned long tstate = regs->tstate; u32 insn; - siginfo_t info; if (notify_die(DIE_TRAP, "illegal instruction", regs, 0, 0x10, SIGILL) == NOTIFY_STOP) - return; + goto out; if (tstate & TSTATE_PRIV) die_if_kernel("Kernel illegal instruction", regs); @@ -2431,14 +2585,14 @@ void do_illegal_instruction(struct pt_regs *regs) if (get_user(insn, (u32 __user *) pc) != -EFAULT) { if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ { if (handle_popc(insn, regs)) - return; + goto out; } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ { if (handle_ldf_stq(insn, regs)) - return; + goto out; } else if (tlb_type == hypervisor) { if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) { if (!vis_emul(regs, insn)) - return; + goto out; } else { struct fpustate *f = FPUSTATE; @@ -2448,44 +2602,37 @@ void do_illegal_instruction(struct pt_regs *regs) * Trap in the %fsr to unimplemented_FPop. */ if (do_mathemu(regs, f, true)) - return; + goto out; } } } - info.si_signo = SIGILL; - info.si_errno = 0; - info.si_code = ILL_ILLOPC; - info.si_addr = (void __user *)pc; - info.si_trapno = 0; - force_sig_info(SIGILL, &info, current); + force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)pc); +out: + exception_exit(prev_state); } -extern void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn); - void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) { - siginfo_t info; + enum ctx_state prev_state = exception_enter(); if (notify_die(DIE_TRAP, "memory address unaligned", regs, 0, 0x34, SIGSEGV) == NOTIFY_STOP) - return; + goto out; if (regs->tstate & TSTATE_PRIV) { kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc)); - return; + goto out; } - info.si_signo = SIGBUS; - info.si_errno = 0; - info.si_code = BUS_ADRALN; - info.si_addr = (void __user *)sfar; - info.si_trapno = 0; - force_sig_info(SIGBUS, &info, current); + if (is_no_fault_exception(regs)) + return; + + force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)sfar); +out: + exception_exit(prev_state); } void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) { - siginfo_t info; - if (notify_die(DIE_TRAP, "memory address unaligned", regs, 0, 0x34, SIGSEGV) == NOTIFY_STOP) return; @@ -2494,32 +2641,74 @@ void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_c kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc)); return; } - info.si_signo = SIGBUS; - info.si_errno = 0; - info.si_code = BUS_ADRALN; - info.si_addr = (void __user *) addr; - info.si_trapno = 0; - force_sig_info(SIGBUS, &info, current); + if (is_no_fault_exception(regs)) + return; + + force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) addr); +} + +/* sun4v_mem_corrupt_detect_precise() - Handle precise exception on an ADI + * tag mismatch. + * + * ADI version tag mismatch on a load from memory always results in a + * precise exception. Tag mismatch on a store to memory will result in + * precise exception if MCDPER or PMCDPER is set to 1. + */ +void sun4v_mem_corrupt_detect_precise(struct pt_regs *regs, unsigned long addr, + unsigned long context) +{ + if (notify_die(DIE_TRAP, "memory corruption precise exception", regs, + 0, 0x8, SIGSEGV) == NOTIFY_STOP) + return; + + if (regs->tstate & TSTATE_PRIV) { + /* MCD exception could happen because the task was running + * a system call with MCD enabled and passed a non-versioned + * pointer or pointer with bad version tag to the system + * call. + */ + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->tpc); + if (entry) { + /* Looks like a bad syscall parameter */ +#ifdef DEBUG_EXCEPTIONS + pr_emerg("Exception: PC<%016lx> faddr<UNKNOWN>\n", + regs->tpc); + pr_emerg("EX_TABLE: insn<%016lx> fixup<%016lx>\n", + regs->tpc, entry->fixup); +#endif + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + return; + } + pr_emerg("%s: ADDR[%016lx] CTX[%lx], going.\n", + __func__, addr, context); + die_if_kernel("MCD precise", regs); + } + + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + force_sig_fault(SIGSEGV, SEGV_ADIPERR, (void __user *)addr); } void do_privop(struct pt_regs *regs) { - siginfo_t info; + enum ctx_state prev_state = exception_enter(); if (notify_die(DIE_TRAP, "privileged operation", regs, 0, 0x11, SIGILL) == NOTIFY_STOP) - return; + goto out; if (test_thread_flag(TIF_32BIT)) { regs->tpc &= 0xffffffff; regs->tnpc &= 0xffffffff; } - info.si_signo = SIGILL; - info.si_errno = 0; - info.si_code = ILL_PRVOPC; - info.si_addr = (void __user *)regs->tpc; - info.si_trapno = 0; - force_sig_info(SIGILL, &info, current); + force_sig_fault(SIGILL, ILL_PRVOPC, (void __user *)regs->tpc); +out: + exception_exit(prev_state); } void do_privact(struct pt_regs *regs) @@ -2530,99 +2719,88 @@ void do_privact(struct pt_regs *regs) /* Trap level 1 stuff or other traps we should never see... */ void do_cee(struct pt_regs *regs) { + exception_enter(); die_if_kernel("TL0: Cache Error Exception", regs); } -void do_cee_tl1(struct pt_regs *regs) -{ - dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); - die_if_kernel("TL1: Cache Error Exception", regs); -} - -void do_dae_tl1(struct pt_regs *regs) -{ - dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); - die_if_kernel("TL1: Data Access Exception", regs); -} - -void do_iae_tl1(struct pt_regs *regs) -{ - dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); - die_if_kernel("TL1: Instruction Access Exception", regs); -} - void do_div0_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: DIV0 Exception", regs); } -void do_fpdis_tl1(struct pt_regs *regs) -{ - dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); - die_if_kernel("TL1: FPU Disabled", regs); -} - void do_fpieee_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: FPU IEEE Exception", regs); } void do_fpother_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: FPU Other Exception", regs); } void do_ill_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: Illegal Instruction Exception", regs); } void do_irq_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: IRQ Exception", regs); } void do_lddfmna_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: LDDF Exception", regs); } void do_stdfmna_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: STDF Exception", regs); } void do_paw(struct pt_regs *regs) { + exception_enter(); die_if_kernel("TL0: Phys Watchpoint Exception", regs); } void do_paw_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: Phys Watchpoint Exception", regs); } void do_vaw(struct pt_regs *regs) { + exception_enter(); die_if_kernel("TL0: Virt Watchpoint Exception", regs); } void do_vaw_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: Virt Watchpoint Exception", regs); } void do_tof_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: Tag Overflow Exception", regs); } @@ -2638,6 +2816,7 @@ void do_getpsr(struct pt_regs *regs) } } +u64 cpu_mondo_counter[NR_CPUS] = {0}; struct trap_per_cpu trap_block[NR_CPUS]; EXPORT_SYMBOL(trap_block); @@ -2670,8 +2849,6 @@ void __init trap_init(void) fault_address) || TI_KREGS != offsetof(struct thread_info, kregs) || TI_UTRAPS != offsetof(struct thread_info, utraps) || - TI_EXEC_DOMAIN != offsetof(struct thread_info, - exec_domain) || TI_REG_WINDOW != offsetof(struct thread_info, reg_window) || TI_RWIN_SPTRS != offsetof(struct thread_info, @@ -2681,10 +2858,6 @@ void __init trap_init(void) TI_PRE_COUNT != offsetof(struct thread_info, preempt_count) || TI_NEW_CHILD != offsetof(struct thread_info, new_child) || - TI_CURRENT_DS != offsetof(struct thread_info, - current_ds) || - TI_RESTART_BLOCK != offsetof(struct thread_info, - restart_block) || TI_KUNA_REGS != offsetof(struct thread_info, kern_una_regs) || TI_KUNA_INSN != offsetof(struct thread_info, @@ -2747,6 +2920,6 @@ void __init trap_init(void) /* Attach to the address space of init_task. On SMP we * do this in smp.c:smp_callin for other cpus. */ - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; } diff --git a/arch/sparc/kernel/tsb.S b/arch/sparc/kernel/tsb.S index a313e4a9399b..eaed39ce8938 100644 --- a/arch/sparc/kernel/tsb.S +++ b/arch/sparc/kernel/tsb.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* tsb.S: Sparc64 TSB table handling. * * Copyright (C) 2006 David S. Miller <davem@davemloft.net> @@ -29,13 +30,17 @@ */ tsb_miss_dtlb: mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_DMMU, %g4 + srlx %g4, PAGE_SHIFT, %g4 ba,pt %xcc, tsb_miss_page_table_walk - ldxa [%g4] ASI_DMMU, %g4 + sllx %g4, PAGE_SHIFT, %g4 tsb_miss_itlb: mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_IMMU, %g4 + srlx %g4, PAGE_SHIFT, %g4 ba,pt %xcc, tsb_miss_page_table_walk - ldxa [%g4] ASI_IMMU, %g4 + sllx %g4, PAGE_SHIFT, %g4 /* At this point we have: * %g1 -- PAGE_SIZE TSB entry address @@ -75,7 +80,7 @@ tsb_miss_page_table_walk: mov 512, %g7 andn %g5, 0x7, %g5 sllx %g7, %g6, %g7 - srlx %g4, HPAGE_SHIFT, %g6 + srlx %g4, REAL_HPAGE_SHIFT, %g6 sub %g7, 1, %g7 and %g6, %g7, %g6 sllx %g6, 4, %g6 @@ -113,26 +118,11 @@ tsb_miss_page_table_walk_sun4v_fastpath: /* Valid PTE is now in %g5. */ #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) -661: sethi %uhi(_PAGE_SZALL_4U), %g7 + sethi %uhi(_PAGE_PMD_HUGE | _PAGE_PUD_HUGE), %g7 sllx %g7, 32, %g7 - .section .sun4v_2insn_patch, "ax" - .word 661b - mov _PAGE_SZALL_4V, %g7 - nop - .previous - - and %g5, %g7, %g2 - -661: sethi %uhi(_PAGE_SZHUGE_4U), %g7 - sllx %g7, 32, %g7 - .section .sun4v_2insn_patch, "ax" - .word 661b - mov _PAGE_SZHUGE_4V, %g7 - nop - .previous - cmp %g2, %g7 - bne,pt %xcc, 60f + andcc %g5, %g7, %g0 + be,pt %xcc, 60f nop /* It is a huge page, use huge page TSB entry address we @@ -162,10 +152,10 @@ tsb_miss_page_table_walk_sun4v_fastpath: nop .previous - rdpr %tl, %g3 - cmp %g3, 1 + rdpr %tl, %g7 + cmp %g7, 1 bne,pn %xcc, winfix_trampoline - nop + mov %g3, %g4 ba,pt %xcc, etrap rd %pc, %g7 call hugetlb_setup @@ -284,6 +274,10 @@ tsb_do_dtlb_fault: nop .previous + /* Clear context ID bits. */ + srlx %g5, PAGE_SHIFT, %g5 + sllx %g5, PAGE_SHIFT, %g5 + be,pt %xcc, sparc64_realfault_common mov FAULT_CODE_DTLB, %g4 ba,pt %xcc, winfix_trampoline @@ -367,6 +361,7 @@ tsb_flush: * %o1: TSB base config pointer * %o2: TSB huge config pointer, or NULL if none * %o3: Hypervisor TSB descriptor physical address + * %o4: Secondary context to load, if non-zero * * We have to run this whole thing with interrupts * disabled so that the current cpu doesn't change @@ -379,6 +374,17 @@ __tsb_context_switch: rdpr %pstate, %g1 wrpr %g1, PSTATE_IE, %pstate + brz,pn %o4, 1f + mov SECONDARY_CONTEXT, %o5 + +661: stxa %o4, [%o5] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %o4, [%o5] ASI_MMU + .previous + flush %g6 + +1: TRAP_LOAD_TRAP_BLOCK(%g2, %g3) stx %o0, [%g2 + TRAP_PER_CPU_PGD_PADDR] @@ -462,13 +468,16 @@ __tsb_context_switch: .type copy_tsb,#function copy_tsb: /* %o0=old_tsb_base, %o1=old_tsb_size * %o2=new_tsb_base, %o3=new_tsb_size + * %o4=page_size_shift */ sethi %uhi(TSB_PASS_BITS), %g7 srlx %o3, 4, %o3 - add %o0, %o1, %g1 /* end of old tsb */ + add %o0, %o1, %o1 /* end of old tsb */ sllx %g7, 32, %g7 sub %o3, 1, %o3 /* %o3 == new tsb hash mask */ + mov %o4, %g1 /* page_size_shift */ + 661: prefetcha [%o0] ASI_N, #one_read .section .tsb_phys_patch, "ax" .word 661b @@ -493,9 +502,9 @@ copy_tsb: /* %o0=old_tsb_base, %o1=old_tsb_size /* This can definitely be computed faster... */ srlx %o0, 4, %o5 /* Build index */ and %o5, 511, %o5 /* Mask index */ - sllx %o5, PAGE_SHIFT, %o5 /* Put into vaddr position */ + sllx %o5, %g1, %o5 /* Put into vaddr position */ or %o4, %o5, %o4 /* Full VADDR. */ - srlx %o4, PAGE_SHIFT, %o4 /* Shift down to create index */ + srlx %o4, %g1, %o4 /* Shift down to create index */ and %o4, %o3, %o4 /* Mask with new_tsb_nents-1 */ sllx %o4, 4, %o4 /* Shift back up into tsb ent offset */ TSB_STORE(%o2 + %o4, %g2) /* Store TAG */ @@ -503,7 +512,7 @@ copy_tsb: /* %o0=old_tsb_base, %o1=old_tsb_size TSB_STORE(%o2 + %o4, %g3) /* Store TTE */ 80: add %o0, 16, %o0 - cmp %o0, %g1 + cmp %o0, %o1 bne,pt %xcc, 90b nop diff --git a/arch/sparc/kernel/ttable_32.S b/arch/sparc/kernel/ttable_32.S index 8a7a96ca676f..e79fd786fbbb 100644 --- a/arch/sparc/kernel/ttable_32.S +++ b/arch/sparc/kernel/ttable_32.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* The Sparc trap table, bootloader gives us control at _start. */ __HEAD diff --git a/arch/sparc/kernel/ttable_64.S b/arch/sparc/kernel/ttable_64.S index c6dfdaa29e20..86e737e59c7e 100644 --- a/arch/sparc/kernel/ttable_64.S +++ b/arch/sparc/kernel/ttable_64.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* ttable.S: Sparc V9 Trap Table(s) with SpitFire/Cheetah/SUN4V extensions. * * Copyright (C) 1996, 2001, 2006 David S. Miller (davem@davemloft.net) @@ -25,8 +26,10 @@ tl0_ill: membar #Sync TRAP_7INSNS(do_illegal_instruction) tl0_privop: TRAP(do_privop) tl0_resv012: BTRAP(0x12) BTRAP(0x13) BTRAP(0x14) BTRAP(0x15) BTRAP(0x16) BTRAP(0x17) -tl0_resv018: BTRAP(0x18) BTRAP(0x19) BTRAP(0x1a) BTRAP(0x1b) BTRAP(0x1c) BTRAP(0x1d) -tl0_resv01e: BTRAP(0x1e) BTRAP(0x1f) +tl0_resv018: BTRAP(0x18) BTRAP(0x19) +tl0_mcd: SUN4V_MCD_PRECISE +tl0_resv01b: BTRAP(0x1b) +tl0_resv01c: BTRAP(0x1c) BTRAP(0x1d) BTRAP(0x1e) BTRAP(0x1f) tl0_fpdis: TRAP_NOSAVE(do_fpdis) tl0_fpieee: TRAP_SAVEFPU(do_fpieee) tl0_fpother: TRAP_NOSAVE(do_fpother_check_fitos) @@ -50,7 +53,7 @@ tl0_resv03e: BTRAP(0x3e) BTRAP(0x3f) BTRAP(0x40) tl0_irq1: TRAP_IRQ(smp_call_function_client, 1) tl0_irq2: TRAP_IRQ(smp_receive_signal_client, 2) tl0_irq3: TRAP_IRQ(smp_penguin_jailcell, 3) -tl0_irq4: TRAP_IRQ(smp_new_mmu_context_version_client, 4) +tl0_irq4: BTRAP(0x44) #else tl0_irq1: BTRAP(0x41) tl0_irq2: BTRAP(0x42) @@ -165,7 +168,7 @@ tl0_resv169: BTRAP(0x169) BTRAP(0x16a) BTRAP(0x16b) BTRAP(0x16c) tl0_linux64: LINUX_64BIT_SYSCALL_TRAP tl0_gsctx: TRAP(sparc64_get_context) TRAP(sparc64_set_context) tl0_resv170: KPROBES_TRAP(0x170) KPROBES_TRAP(0x171) KGDB_TRAP(0x172) -tl0_resv173: BTRAP(0x173) BTRAP(0x174) BTRAP(0x175) BTRAP(0x176) BTRAP(0x177) +tl0_resv173: UPROBES_TRAP(0x173) UPROBES_TRAP(0x174) BTRAP(0x175) BTRAP(0x176) BTRAP(0x177) tl0_resv178: BTRAP(0x178) BTRAP(0x179) BTRAP(0x17a) BTRAP(0x17b) BTRAP(0x17c) tl0_resv17d: BTRAP(0x17d) BTRAP(0x17e) BTRAP(0x17f) #define BTRAPS(x) BTRAP(x) BTRAP(x+1) BTRAP(x+2) BTRAP(x+3) BTRAP(x+4) BTRAP(x+5) BTRAP(x+6) BTRAP(x+7) diff --git a/arch/sparc/kernel/una_asm_32.S b/arch/sparc/kernel/una_asm_32.S index 8f096e84a937..f8bf839289fb 100644 --- a/arch/sparc/kernel/una_asm_32.S +++ b/arch/sparc/kernel/una_asm_32.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* una_asm.S: Kernel unaligned trap assembler helpers. * * Copyright (C) 1996,2005,2008 David S. Miller (davem@davemloft.net) diff --git a/arch/sparc/kernel/una_asm_64.S b/arch/sparc/kernel/una_asm_64.S index 1c8d33228b2a..e256f395e9f6 100644 --- a/arch/sparc/kernel/una_asm_64.S +++ b/arch/sparc/kernel/una_asm_64.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* una_asm.S: Kernel unaligned trap assembler helpers. * * Copyright (C) 1996,2005 David S. Miller (davem@davemloft.net) diff --git a/arch/sparc/kernel/unaligned_32.c b/arch/sparc/kernel/unaligned_32.c index c0ec89786193..455f0258c745 100644 --- a/arch/sparc/kernel/unaligned_32.c +++ b/arch/sparc/kernel/unaligned_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * unaligned.c: Unaligned load/store trap handling with special * cases for the kernel to do them more quickly. @@ -8,13 +9,18 @@ #include <linux/kernel.h> -#include <linux/sched.h> +#include <linux/sched/signal.h> #include <linux/mm.h> #include <asm/ptrace.h> #include <asm/processor.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <linux/smp.h> #include <linux/perf_event.h> +#include <linux/extable.h> + +#include <asm/setup.h> + +#include "kernel.h" enum direction { load, /* ld, ldd, ldh, ldsh */ @@ -162,7 +168,7 @@ unsigned long safe_compute_effective_address(struct pt_regs *regs, /* This is just to make gcc think panic does return... */ static void unaligned_panic(char *str) { - panic(str); + panic("%s", str); } /* una_asm.S */ @@ -208,10 +214,10 @@ static inline int ok_for_kernel(unsigned int insn) static void kernel_mna_trap_fault(struct pt_regs *regs, unsigned int insn) { - unsigned long g2 = regs->u_regs [UREG_G2]; - unsigned long fixup = search_extables_range(regs->pc, &g2); + const struct exception_table_entry *entry; - if (!fixup) { + entry = search_exception_tables(regs->pc); + if (!entry) { unsigned long address = compute_effective_address(regs, insn); if(address < PAGE_SIZE) { printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference in mna handler"); @@ -227,9 +233,8 @@ static void kernel_mna_trap_fault(struct pt_regs *regs, unsigned int insn) die_if_kernel("Oops", regs); /* Not reached */ } - regs->pc = fixup; + regs->pc = entry->fixup; regs->npc = regs->pc + 4; - regs->u_regs [UREG_G2] = g2; } asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) @@ -269,109 +274,9 @@ asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) } } -static inline int ok_for_user(struct pt_regs *regs, unsigned int insn, - enum direction dir) -{ - unsigned int reg; - int check = (dir == load) ? VERIFY_READ : VERIFY_WRITE; - int size = ((insn >> 19) & 3) == 3 ? 8 : 4; - - if ((regs->pc | regs->npc) & 3) - return 0; - - /* Must access_ok() in all the necessary places. */ -#define WINREG_ADDR(regnum) \ - ((void __user *)(((unsigned long *)regs->u_regs[UREG_FP])+(regnum))) - - reg = (insn >> 25) & 0x1f; - if (reg >= 16) { - if (!access_ok(check, WINREG_ADDR(reg - 16), size)) - return -EFAULT; - } - reg = (insn >> 14) & 0x1f; - if (reg >= 16) { - if (!access_ok(check, WINREG_ADDR(reg - 16), size)) - return -EFAULT; - } - if (!(insn & 0x2000)) { - reg = (insn & 0x1f); - if (reg >= 16) { - if (!access_ok(check, WINREG_ADDR(reg - 16), size)) - return -EFAULT; - } - } -#undef WINREG_ADDR - return 0; -} - -static void user_mna_trap_fault(struct pt_regs *regs, unsigned int insn) -{ - siginfo_t info; - - info.si_signo = SIGBUS; - info.si_errno = 0; - info.si_code = BUS_ADRALN; - info.si_addr = (void __user *)safe_compute_effective_address(regs, insn); - info.si_trapno = 0; - send_sig_info(SIGBUS, &info, current); -} - asmlinkage void user_unaligned_trap(struct pt_regs *regs, unsigned int insn) { - enum direction dir; - - if(!(current->thread.flags & SPARC_FLAG_UNALIGNED) || - (((insn >> 30) & 3) != 3)) - goto kill_user; - dir = decode_direction(insn); - if(!ok_for_user(regs, insn, dir)) { - goto kill_user; - } else { - int err, size = decode_access_size(insn); - unsigned long addr; - - if(floating_point_load_or_store_p(insn)) { - printk("User FPU load/store unaligned unsupported.\n"); - goto kill_user; - } - - addr = compute_effective_address(regs, insn); - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); - switch(dir) { - case load: - err = do_int_load(fetch_reg_addr(((insn>>25)&0x1f), - regs), - size, (unsigned long *) addr, - decode_signedness(insn)); - break; - - case store: - err = do_int_store(((insn>>25)&0x1f), size, - (unsigned long *) addr, regs); - break; - - case both: - /* - * This was supported in 2.4. However, we question - * the value of SWAP instruction across word boundaries. - */ - printk("Unaligned SWAP unsupported.\n"); - err = -EFAULT; - break; - - default: - unaligned_panic("Impossible user unaligned trap."); - goto out; - } - if (err) - goto kill_user; - else - advance(regs); - goto out; - } - -kill_user: - user_mna_trap_fault(regs, insn); -out: - ; + send_sig_fault(SIGBUS, BUS_ADRALN, + (void __user *)safe_compute_effective_address(regs, insn), + current); } diff --git a/arch/sparc/kernel/unaligned_64.c b/arch/sparc/kernel/unaligned_64.c index 8201c25e7669..23db2efda570 100644 --- a/arch/sparc/kernel/unaligned_64.c +++ b/arch/sparc/kernel/unaligned_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * unaligned.c: Unaligned load/store trap handling with special * cases for the kernel to do them more quickly. @@ -11,18 +12,23 @@ #include <linux/kernel.h> #include <linux/sched.h> #include <linux/mm.h> -#include <linux/module.h> +#include <linux/extable.h> #include <asm/asi.h> #include <asm/ptrace.h> #include <asm/pstate.h> #include <asm/processor.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <linux/smp.h> #include <linux/bitops.h> #include <linux/perf_event.h> #include <linux/ratelimit.h> +#include <linux/context_tracking.h> #include <asm/fpumacro.h> #include <asm/cacheflush.h> +#include <asm/setup.h> + +#include "entry.h" +#include "kernel.h" enum direction { load, /* ld, ldd, ldh, ldsh */ @@ -163,17 +169,23 @@ static unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs) unsigned long compute_effective_address(struct pt_regs *regs, unsigned int insn, unsigned int rd) { + int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; unsigned int rs1 = (insn >> 14) & 0x1f; unsigned int rs2 = insn & 0x1f; - int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; + unsigned long addr; if (insn & 0x2000) { maybe_flush_windows(rs1, 0, rd, from_kernel); - return (fetch_reg(rs1, regs) + sign_extend_imm13(insn)); + addr = (fetch_reg(rs1, regs) + sign_extend_imm13(insn)); } else { maybe_flush_windows(rs1, rs2, rd, from_kernel); - return (fetch_reg(rs1, regs) + fetch_reg(rs2, regs)); + addr = (fetch_reg(rs1, regs) + fetch_reg(rs2, regs)); } + + if (!from_kernel && test_thread_flag(TIF_32BIT)) + addr &= 0xffffffff; + + return addr; } /* This is just to make gcc think die_if_kernel does return... */ @@ -198,8 +210,8 @@ static inline int do_int_store(int reg_num, int size, unsigned long *dst_addr, if (size == 16) { size = 8; zero = (((long)(reg_num ? - (unsigned)fetch_reg(reg_num, regs) : 0)) << 32) | - (unsigned)fetch_reg(reg_num + 1, regs); + (unsigned int)fetch_reg(reg_num, regs) : 0)) << 32) | + (unsigned int)fetch_reg(reg_num + 1, regs); } else if (reg_num) { src_val_p = fetch_reg_addr(reg_num, regs); } @@ -418,9 +430,6 @@ int handle_popc(u32 insn, struct pt_regs *regs) extern void do_fpother(struct pt_regs *regs); extern void do_privact(struct pt_regs *regs); -extern void spitfire_data_access_exception(struct pt_regs *regs, - unsigned long sfsr, - unsigned long sfar); extern void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx); @@ -428,24 +437,26 @@ extern void sun4v_data_access_exception(struct pt_regs *regs, int handle_ldf_stq(u32 insn, struct pt_regs *regs) { unsigned long addr = compute_effective_address(regs, insn, 0); - int freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20); + int freg; struct fpustate *f = FPUSTATE; int asi = decode_asi(insn, regs); - int flag = (freg < 32) ? FPRS_DL : FPRS_DU; + int flag; perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); save_and_clear_fpu(); current_thread_info()->xfsr[0] &= ~0x1c000; - if (freg & 3) { - current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */; - do_fpother(regs); - return 0; - } if (insn & 0x200000) { /* STQ */ u64 first = 0, second = 0; + freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20); + flag = (freg < 32) ? FPRS_DL : FPRS_DU; + if (freg & 3) { + current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */; + do_fpother(regs); + return 0; + } if (current_thread_info()->fpsaved[0] & flag) { first = *(u64 *)&f->regs[freg]; second = *(u64 *)&f->regs[freg+2]; @@ -505,6 +516,12 @@ int handle_ldf_stq(u32 insn, struct pt_regs *regs) case 0x100000: size = 4; break; default: size = 2; break; } + if (size == 1) + freg = (insn >> 25) & 0x1f; + else + freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20); + flag = (freg < 32) ? FPRS_DL : FPRS_DU; + for (i = 0; i < size; i++) data[i] = 0; @@ -578,6 +595,7 @@ void handle_ld_nf(u32 insn, struct pt_regs *regs) void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) { + enum ctx_state prev_state = exception_enter(); unsigned long pc = regs->tpc; unsigned long tstate = regs->tstate; u32 insn; @@ -632,13 +650,16 @@ daex: sun4v_data_access_exception(regs, sfar, sfsr); else spitfire_data_access_exception(regs, sfsr, sfar); - return; + goto out; } advance(regs); +out: + exception_exit(prev_state); } void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) { + enum ctx_state prev_state = exception_enter(); unsigned long pc = regs->tpc; unsigned long tstate = regs->tstate; u32 insn; @@ -680,7 +701,9 @@ daex: sun4v_data_access_exception(regs, sfar, sfsr); else spitfire_data_access_exception(regs, sfsr, sfar); - return; + goto out; } advance(regs); +out: + exception_exit(prev_state); } diff --git a/arch/sparc/kernel/uprobes.c b/arch/sparc/kernel/uprobes.c new file mode 100644 index 000000000000..305017bec164 --- /dev/null +++ b/arch/sparc/kernel/uprobes.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * User-space Probes (UProbes) for sparc + * + * Copyright (C) 2013 Oracle Inc. + * + * Authors: + * Jose E. Marchesi <jose.marchesi@oracle.com> + * Eric Saint Etienne <eric.saint.etienne@oracle.com> + */ + +#include <linux/kernel.h> +#include <linux/highmem.h> +#include <linux/uprobes.h> +#include <linux/uaccess.h> +#include <linux/sched.h> /* For struct task_struct */ +#include <linux/kdebug.h> + +#include <asm/cacheflush.h> + +#include "kernel.h" + +/* Compute the address of the breakpoint instruction and return it. + * + * Note that uprobe_get_swbp_addr is defined as a weak symbol in + * kernel/events/uprobe.c. + */ +unsigned long uprobe_get_swbp_addr(struct pt_regs *regs) +{ + return instruction_pointer(regs); +} + +static void copy_to_page(struct page *page, unsigned long vaddr, + const void *src, int len) +{ + void *kaddr = kmap_atomic(page); + + memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len); + kunmap_atomic(kaddr); +} + +/* Fill in the xol area with the probed instruction followed by the + * single-step trap. Some fixups in the copied instruction are + * performed at this point. + * + * Note that uprobe_xol_copy is defined as a weak symbol in + * kernel/events/uprobe.c. + */ +void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, + void *src, unsigned long len) +{ + const u32 stp_insn = UPROBE_STP_INSN; + u32 insn = *(u32 *) src; + + /* Branches annulling their delay slot must be fixed to not do + * so. Clearing the annul bit on these instructions we can be + * sure the single-step breakpoint in the XOL slot will be + * executed. + */ + + u32 op = (insn >> 30) & 0x3; + u32 op2 = (insn >> 22) & 0x7; + + if (op == 0 && + (op2 == 1 || op2 == 2 || op2 == 3 || op2 == 5 || op2 == 6) && + (insn & ANNUL_BIT) == ANNUL_BIT) + insn &= ~ANNUL_BIT; + + copy_to_page(page, vaddr, &insn, len); + copy_to_page(page, vaddr+len, &stp_insn, 4); +} + + +/* Instruction analysis/validity. + * + * This function returns 0 on success or a -ve number on error. + */ +int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, + struct mm_struct *mm, unsigned long addr) +{ + /* Any unsupported instruction? Then return -EINVAL */ + return 0; +} + +/* If INSN is a relative control transfer instruction, return the + * corrected branch destination value. + * + * Note that regs->tpc and regs->tnpc still hold the values of the + * program counters at the time of the single-step trap due to the + * execution of the UPROBE_STP_INSN at utask->xol_vaddr + 4. + * + */ +static unsigned long relbranch_fixup(u32 insn, struct uprobe_task *utask, + struct pt_regs *regs) +{ + /* Branch not taken, no mods necessary. */ + if (regs->tnpc == regs->tpc + 0x4UL) + return utask->autask.saved_tnpc + 0x4UL; + + /* The three cases are call, branch w/prediction, + * and traditional branch. + */ + if ((insn & 0xc0000000) == 0x40000000 || + (insn & 0xc1c00000) == 0x00400000 || + (insn & 0xc1c00000) == 0x00800000) { + unsigned long real_pc = (unsigned long) utask->vaddr; + unsigned long ixol_addr = utask->xol_vaddr; + + /* The instruction did all the work for us + * already, just apply the offset to the correct + * instruction location. + */ + return (real_pc + (regs->tnpc - ixol_addr)); + } + + /* It is jmpl or some other absolute PC modification instruction, + * leave NPC as-is. + */ + return regs->tnpc; +} + +/* If INSN is an instruction which writes its PC location + * into a destination register, fix that up. + */ +static int retpc_fixup(struct pt_regs *regs, u32 insn, + unsigned long real_pc) +{ + unsigned long *slot = NULL; + int rc = 0; + + /* Simplest case is 'call', which always uses %o7 */ + if ((insn & 0xc0000000) == 0x40000000) + slot = ®s->u_regs[UREG_I7]; + + /* 'jmpl' encodes the register inside of the opcode */ + if ((insn & 0xc1f80000) == 0x81c00000) { + unsigned long rd = ((insn >> 25) & 0x1f); + + if (rd <= 15) { + slot = ®s->u_regs[rd]; + } else { + unsigned long fp = regs->u_regs[UREG_FP]; + /* Hard case, it goes onto the stack. */ + flushw_all(); + + rd -= 16; + if (test_thread_64bit_stack(fp)) { + unsigned long __user *uslot = + (unsigned long __user *) (fp + STACK_BIAS) + rd; + rc = __put_user(real_pc, uslot); + } else { + unsigned int __user *uslot = (unsigned int + __user *) fp + rd; + rc = __put_user((u32) real_pc, uslot); + } + } + } + if (slot != NULL) + *slot = real_pc; + return rc; +} + +/* Single-stepping can be avoided for certain instructions: NOPs and + * instructions that can be emulated. This function determines + * whether the instruction where the uprobe is installed falls in one + * of these cases and emulates it. + * + * This function returns true if the single-stepping can be skipped, + * false otherwise. + */ +bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + /* We currently only emulate NOP instructions. + */ + + if (auprobe->ixol == (1 << 24)) { + regs->tnpc += 4; + regs->tpc += 4; + return true; + } + + return false; +} + +/* Prepare to execute out of line. At this point + * current->utask->xol_vaddr points to an allocated XOL slot properly + * initialized with the original instruction and the single-stepping + * trap instruction. + * + * This function returns 0 on success, any other number on error. + */ +int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + struct arch_uprobe_task *autask = ¤t->utask->autask; + + /* Save the current program counters so they can be restored + * later. + */ + autask->saved_tpc = regs->tpc; + autask->saved_tnpc = regs->tnpc; + + /* Adjust PC and NPC so the first instruction in the XOL slot + * will be executed by the user task. + */ + instruction_pointer_set(regs, utask->xol_vaddr); + + return 0; +} + +/* Prepare to resume execution after the single-step. Called after + * single-stepping. To avoid the SMP problems that can occur when we + * temporarily put back the original opcode to single-step, we + * single-stepped a copy of the instruction. + * + * This function returns 0 on success, any other number on error. + */ +int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + struct arch_uprobe_task *autask = &utask->autask; + u32 insn = auprobe->ixol; + int rc = 0; + + if (utask->state == UTASK_SSTEP_ACK) { + regs->tnpc = relbranch_fixup(insn, utask, regs); + regs->tpc = autask->saved_tnpc; + rc = retpc_fixup(regs, insn, (unsigned long) utask->vaddr); + } else { + regs->tnpc = utask->vaddr+4; + regs->tpc = autask->saved_tnpc+4; + } + return rc; +} + +/* Handler for uprobe traps. This is called from the traps table and + * triggers the proper die notification. + */ +asmlinkage void uprobe_trap(struct pt_regs *regs, + unsigned long trap_level) +{ + BUG_ON(trap_level != 0x173 && trap_level != 0x174); + + /* We are only interested in user-mode code. Uprobe traps + * shall not be present in kernel code. + */ + if (!user_mode(regs)) { + local_irq_enable(); + bad_trap(regs, trap_level); + return; + } + + /* trap_level == 0x173 --> ta 0x73 + * trap_level == 0x174 --> ta 0x74 + */ + if (notify_die((trap_level == 0x173) ? DIE_BPT : DIE_SSTEP, + (trap_level == 0x173) ? "bpt" : "sstep", + regs, 0, trap_level, SIGTRAP) != NOTIFY_STOP) + bad_trap(regs, trap_level); +} + +/* Callback routine for handling die notifications. +*/ +int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + int ret = NOTIFY_DONE; + struct die_args *args = (struct die_args *)data; + + /* We are only interested in userspace traps */ + if (args->regs && !user_mode(args->regs)) + return NOTIFY_DONE; + + switch (val) { + case DIE_BPT: + if (uprobe_pre_sstep_notifier(args->regs)) + ret = NOTIFY_STOP; + break; + + case DIE_SSTEP: + if (uprobe_post_sstep_notifier(args->regs)) + ret = NOTIFY_STOP; + + default: + break; + } + + return ret; +} + +/* This function gets called when a XOL instruction either gets + * trapped or the thread has a fatal signal, so reset the instruction + * pointer to its probed address. + */ +void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + instruction_pointer_set(regs, utask->vaddr); +} + +/* If xol insn itself traps and generates a signal(Say, + * SIGILL/SIGSEGV/etc), then detect the case where a singlestepped + * instruction jumps back to its own address. + */ +bool arch_uprobe_xol_was_trapped(struct task_struct *t) +{ + return false; +} + +unsigned long +arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, + struct pt_regs *regs) +{ + unsigned long orig_ret_vaddr = regs->u_regs[UREG_I7]; + + regs->u_regs[UREG_I7] = trampoline_vaddr-8; + + return orig_ret_vaddr + 8; +} diff --git a/arch/sparc/kernel/urtt_fill.S b/arch/sparc/kernel/urtt_fill.S new file mode 100644 index 000000000000..e4cee7be5cd0 --- /dev/null +++ b/arch/sparc/kernel/urtt_fill.S @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <asm/thread_info.h> +#include <asm/trap_block.h> +#include <asm/spitfire.h> +#include <asm/ptrace.h> +#include <asm/head.h> + + .text + .align 8 + .globl user_rtt_fill_fixup_common +user_rtt_fill_fixup_common: + rdpr %cwp, %g1 + add %g1, 1, %g1 + wrpr %g1, 0x0, %cwp + + rdpr %wstate, %g2 + sll %g2, 3, %g2 + wrpr %g2, 0x0, %wstate + + /* We know %canrestore and %otherwin are both zero. */ + + sethi %hi(sparc64_kern_pri_context), %g2 + ldx [%g2 + %lo(sparc64_kern_pri_context)], %g2 + mov PRIMARY_CONTEXT, %g1 + +661: stxa %g2, [%g1] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g1] ASI_MMU + .previous + + sethi %hi(KERNBASE), %g1 + flush %g1 + + mov %g4, %l4 + mov %g5, %l5 + brnz,pn %g3, 1f + mov %g3, %l3 + + or %g4, FAULT_CODE_WINFIXUP, %g4 + stb %g4, [%g6 + TI_FAULT_CODE] + stx %g5, [%g6 + TI_FAULT_ADDR] +1: + mov %g6, %l1 + wrpr %g0, 0x0, %tl + +661: nop + .section .sun4v_1insn_patch, "ax" + .word 661b + SET_GL(0) + .previous + +661: wrpr %g0, RTRAP_PSTATE, %pstate + .section .sun_m7_1insn_patch, "ax" + .word 661b + /* Re-enable PSTATE.mcde to maintain ADI security */ + wrpr %g0, RTRAP_PSTATE|PSTATE_MCDE, %pstate + .previous + + mov %l1, %g6 + ldx [%g6 + TI_TASK], %g4 + LOAD_PER_CPU_BASE(%g5, %g6, %g1, %g2, %g3) + + brnz,pn %l3, 1f + nop + + call do_sparc64_fault + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + +1: cmp %g3, 2 + bne,pn %xcc, 2f + nop + + sethi %hi(tlb_type), %g1 + lduw [%g1 + %lo(tlb_type)], %g1 + cmp %g1, 3 + bne,pt %icc, 1f + add %sp, PTREGS_OFF, %o0 + mov %l4, %o2 + call sun4v_do_mna + mov %l5, %o1 + ba,a,pt %xcc, rtrap +1: mov %l4, %o1 + mov %l5, %o2 + call mem_address_unaligned + nop + ba,a,pt %xcc, rtrap + +2: sethi %hi(tlb_type), %g1 + mov %l4, %o1 + lduw [%g1 + %lo(tlb_type)], %g1 + mov %l5, %o2 + cmp %g1, 3 + bne,pt %icc, 1f + add %sp, PTREGS_OFF, %o0 + call sun4v_data_access_exception + nop + ba,a,pt %xcc, rtrap + nop + +1: call spitfire_data_access_exception + nop + ba,a,pt %xcc, rtrap diff --git a/arch/sparc/kernel/utrap.S b/arch/sparc/kernel/utrap.S index b7f0f3f3a909..7a2d9a9bea59 100644 --- a/arch/sparc/kernel/utrap.S +++ b/arch/sparc/kernel/utrap.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ .globl utrap_trap .type utrap_trap,#function utrap_trap: /* %g3=handler,%g4=level */ @@ -11,8 +12,7 @@ utrap_trap: /* %g3=handler,%g4=level */ mov %l4, %o1 call bad_trap add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap invoke_utrap: sllx %g3, 3, %g3 diff --git a/arch/sparc/kernel/vdso.c b/arch/sparc/kernel/vdso.c new file mode 100644 index 000000000000..0e27437eb97b --- /dev/null +++ b/arch/sparc/kernel/vdso.c @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE + * Copyright 2003 Andi Kleen, SuSE Labs. + * + * Thanks to hpa@transmeta.com for some useful hint. + * Special thanks to Ingo Molnar for his early experience with + * a different vsyscall implementation for Linux/IA32 and for the name. + */ + +#include <linux/time.h> +#include <linux/timekeeper_internal.h> + +#include <asm/vvar.h> + +void update_vsyscall_tz(void) +{ + if (unlikely(vvar_data == NULL)) + return; + + vvar_data->tz_minuteswest = sys_tz.tz_minuteswest; + vvar_data->tz_dsttime = sys_tz.tz_dsttime; +} + +void update_vsyscall(struct timekeeper *tk) +{ + struct vvar_data *vdata = vvar_data; + + if (unlikely(vdata == NULL)) + return; + + vvar_write_begin(vdata); + vdata->vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode; + vdata->clock.cycle_last = tk->tkr_mono.cycle_last; + vdata->clock.mask = tk->tkr_mono.mask; + vdata->clock.mult = tk->tkr_mono.mult; + vdata->clock.shift = tk->tkr_mono.shift; + + vdata->wall_time_sec = tk->xtime_sec; + vdata->wall_time_snsec = tk->tkr_mono.xtime_nsec; + + vdata->monotonic_time_sec = tk->xtime_sec + + tk->wall_to_monotonic.tv_sec; + vdata->monotonic_time_snsec = tk->tkr_mono.xtime_nsec + + (tk->wall_to_monotonic.tv_nsec << + tk->tkr_mono.shift); + + while (vdata->monotonic_time_snsec >= + (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { + vdata->monotonic_time_snsec -= + ((u64)NSEC_PER_SEC) << tk->tkr_mono.shift; + vdata->monotonic_time_sec++; + } + + vdata->wall_time_coarse_sec = tk->xtime_sec; + vdata->wall_time_coarse_nsec = + (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); + + vdata->monotonic_time_coarse_sec = + vdata->wall_time_coarse_sec + tk->wall_to_monotonic.tv_sec; + vdata->monotonic_time_coarse_nsec = + vdata->wall_time_coarse_nsec + tk->wall_to_monotonic.tv_nsec; + + while (vdata->monotonic_time_coarse_nsec >= NSEC_PER_SEC) { + vdata->monotonic_time_coarse_nsec -= NSEC_PER_SEC; + vdata->monotonic_time_coarse_sec++; + } + + vvar_write_end(vdata); +} diff --git a/arch/sparc/kernel/vio.c b/arch/sparc/kernel/vio.c index 8647fcc5ca6c..1a1a9d6b8f2e 100644 --- a/arch/sparc/kernel/vio.c +++ b/arch/sparc/kernel/vio.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* vio.c: Virtual I/O channel devices probing infrastructure. * * Copyright (c) 2003-2005 IBM Corp. @@ -45,10 +46,18 @@ static const struct vio_device_id *vio_match_device( return NULL; } -static int vio_bus_match(struct device *dev, struct device_driver *drv) +static int vio_hotplug(const struct device *dev, struct kobj_uevent_env *env) +{ + const struct vio_dev *vio_dev = to_vio_dev(dev); + + add_uevent_var(env, "MODALIAS=vio:T%sS%s", vio_dev->type, vio_dev->compat); + return 0; +} + +static int vio_bus_match(struct device *dev, const struct device_driver *drv) { struct vio_dev *vio_dev = to_vio_dev(dev); - struct vio_driver *vio_drv = to_vio_driver(drv); + const struct vio_driver *vio_drv = to_vio_driver(drv); const struct vio_device_id *matches = vio_drv->id_table; if (!matches) @@ -62,26 +71,42 @@ static int vio_device_probe(struct device *dev) struct vio_dev *vdev = to_vio_dev(dev); struct vio_driver *drv = to_vio_driver(dev->driver); const struct vio_device_id *id; - int error = -ENODEV; - if (drv->probe) { - id = vio_match_device(drv->id_table, vdev); - if (id) - error = drv->probe(vdev, id); + if (!drv->probe) + return -ENODEV; + + id = vio_match_device(drv->id_table, vdev); + if (!id) + return -ENODEV; + + /* alloc irqs (unless the driver specified not to) */ + if (!drv->no_irq) { + if (vdev->tx_irq == 0 && vdev->tx_ino != ~0UL) + vdev->tx_irq = sun4v_build_virq(vdev->cdev_handle, + vdev->tx_ino); + + if (vdev->rx_irq == 0 && vdev->rx_ino != ~0UL) + vdev->rx_irq = sun4v_build_virq(vdev->cdev_handle, + vdev->rx_ino); } - return error; + return drv->probe(vdev, id); } -static int vio_device_remove(struct device *dev) +static void vio_device_remove(struct device *dev) { struct vio_dev *vdev = to_vio_dev(dev); struct vio_driver *drv = to_vio_driver(dev->driver); - if (drv->remove) - return drv->remove(vdev); + if (drv->remove) { + /* + * Ideally, we would remove/deallocate tx/rx virqs + * here - however, there are currently no support + * routines to do so at the moment. TBD + */ - return 1; + drv->remove(vdev); + } } static ssize_t devspec_show(struct device *dev, @@ -97,6 +122,7 @@ static ssize_t devspec_show(struct device *dev, return sprintf(buf, "%s\n", str); } +static DEVICE_ATTR_RO(devspec); static ssize_t type_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -104,16 +130,29 @@ static ssize_t type_show(struct device *dev, struct vio_dev *vdev = to_vio_dev(dev); return sprintf(buf, "%s\n", vdev->type); } +static DEVICE_ATTR_RO(type); -static struct device_attribute vio_dev_attrs[] = { - __ATTR_RO(devspec), - __ATTR_RO(type), - __ATTR_NULL -}; +static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + const struct vio_dev *vdev = to_vio_dev(dev); + + return sprintf(buf, "vio:T%sS%s\n", vdev->type, vdev->compat); +} +static DEVICE_ATTR_RO(modalias); -static struct bus_type vio_bus_type = { +static struct attribute *vio_dev_attrs[] = { + &dev_attr_devspec.attr, + &dev_attr_type.attr, + &dev_attr_modalias.attr, + NULL, + }; +ATTRIBUTE_GROUPS(vio_dev); + +static const struct bus_type vio_bus_type = { .name = "vio", - .dev_attrs = vio_dev_attrs, + .dev_groups = vio_dev_groups, + .uevent = vio_hotplug, .match = vio_bus_match, .probe = vio_device_probe, .remove = vio_device_remove, @@ -152,7 +191,7 @@ show_pciobppath_attr(struct device *dev, struct device_attribute *attr, vdev = to_vio_dev(dev); dp = vdev->dp; - return snprintf (buf, PAGE_SIZE, "%s\n", dp->full_name); + return scnprintf(buf, PAGE_SIZE, "%pOF\n", dp); } static DEVICE_ATTR(obppath, S_IRUSR | S_IRGRP | S_IROTH, @@ -163,11 +202,59 @@ static struct device_node *cdev_node; static struct vio_dev *root_vdev; static u64 cdev_cfg_handle; +static const u64 *vio_cfg_handle(struct mdesc_handle *hp, u64 node) +{ + const u64 *cfg_handle = NULL; + u64 a; + + mdesc_for_each_arc(a, hp, node, MDESC_ARC_TYPE_BACK) { + u64 target; + + target = mdesc_arc_target(hp, a); + cfg_handle = mdesc_get_property(hp, target, + "cfg-handle", NULL); + if (cfg_handle) + break; + } + + return cfg_handle; +} + +/** + * vio_vdev_node() - Find VDEV node in MD + * @hp: Handle to the MD + * @vdev: Pointer to VDEV + * + * Find the node in the current MD which matches the given vio_dev. This + * must be done dynamically since the node value can change if the MD + * is updated. + * + * NOTE: the MD must be locked, using mdesc_grab(), when calling this routine + * + * Return: The VDEV node in MDESC + */ +u64 vio_vdev_node(struct mdesc_handle *hp, struct vio_dev *vdev) +{ + u64 node; + + if (vdev == NULL) + return MDESC_NODE_NULL; + + node = mdesc_get_node(hp, (const char *)vdev->node_name, + &vdev->md_node_info); + + return node; +} +EXPORT_SYMBOL(vio_vdev_node); + static void vio_fill_channel_info(struct mdesc_handle *hp, u64 mp, struct vio_dev *vdev) { u64 a; + vdev->tx_ino = ~0UL; + vdev->rx_ino = ~0UL; + vdev->channel_id = ~0UL; mdesc_for_each_arc(a, hp, mp, MDESC_ARC_TYPE_FWD) { const u64 *chan_id; const u64 *irq; @@ -177,27 +264,38 @@ static void vio_fill_channel_info(struct mdesc_handle *hp, u64 mp, irq = mdesc_get_property(hp, target, "tx-ino", NULL); if (irq) - vdev->tx_irq = sun4v_build_virq(cdev_cfg_handle, *irq); + vdev->tx_ino = *irq; irq = mdesc_get_property(hp, target, "rx-ino", NULL); if (irq) - vdev->rx_irq = sun4v_build_virq(cdev_cfg_handle, *irq); + vdev->rx_ino = *irq; chan_id = mdesc_get_property(hp, target, "id", NULL); if (chan_id) vdev->channel_id = *chan_id; } + + vdev->cdev_handle = cdev_cfg_handle; +} + +int vio_set_intr(unsigned long dev_ino, int state) +{ + int err; + + err = sun4v_vintr_set_valid(cdev_cfg_handle, dev_ino, state); + return err; } +EXPORT_SYMBOL(vio_set_intr); static struct vio_dev *vio_create_one(struct mdesc_handle *hp, u64 mp, + const char *node_name, struct device *parent) { - const char *type, *compat, *bus_id_name; + const char *type, *compat; struct device_node *dp; struct vio_dev *vdev; int err, tlen, clen; const u64 *id, *cfg_handle; - u64 a; type = mdesc_get_property(hp, mp, "device-type", &tlen); if (!type) { @@ -207,7 +305,7 @@ static struct vio_dev *vio_create_one(struct mdesc_handle *hp, u64 mp, tlen = strlen(type) + 1; } } - if (tlen > VIO_MAX_TYPE_LEN) { + if (tlen > VIO_MAX_TYPE_LEN || strlen(type) >= VIO_MAX_TYPE_LEN) { printk(KERN_ERR "VIO: Type string [%s] is too long.\n", type); return NULL; @@ -215,31 +313,7 @@ static struct vio_dev *vio_create_one(struct mdesc_handle *hp, u64 mp, id = mdesc_get_property(hp, mp, "id", NULL); - cfg_handle = NULL; - mdesc_for_each_arc(a, hp, mp, MDESC_ARC_TYPE_BACK) { - u64 target; - - target = mdesc_arc_target(hp, a); - cfg_handle = mdesc_get_property(hp, target, - "cfg-handle", NULL); - if (cfg_handle) - break; - } - - bus_id_name = type; - if (!strcmp(type, "domain-services-port")) - bus_id_name = "ds"; - - /* - * 20 char is the old driver-core name size limit, which is no more. - * This check can probably be removed after review and possible - * adaption of the vio users name length handling. - */ - if (strlen(bus_id_name) >= 20 - 4) { - printk(KERN_ERR "VIO: bus_id_name [%s] is too long.\n", - bus_id_name); - return NULL; - } + cfg_handle = vio_cfg_handle(hp, mp); compat = mdesc_get_property(hp, mp, "device-type", &clen); if (!compat) { @@ -264,22 +338,23 @@ static struct vio_dev *vio_create_one(struct mdesc_handle *hp, u64 mp, memset(vdev->compat, 0, sizeof(vdev->compat)); vdev->compat_len = clen; - vdev->channel_id = ~0UL; - vdev->tx_irq = ~0; - vdev->rx_irq = ~0; + vdev->port_id = ~0UL; + vdev->tx_irq = 0; + vdev->rx_irq = 0; vio_fill_channel_info(hp, mp, vdev); if (!id) { - dev_set_name(&vdev->dev, "%s", bus_id_name); + dev_set_name(&vdev->dev, "%s", type); vdev->dev_no = ~(u64)0; } else if (!cfg_handle) { - dev_set_name(&vdev->dev, "%s-%llu", bus_id_name, *id); + dev_set_name(&vdev->dev, "%s-%llu", type, *id); vdev->dev_no = *id; } else { - dev_set_name(&vdev->dev, "%s-%llu-%llu", bus_id_name, + dev_set_name(&vdev->dev, "%s-%llu-%llu", type, *cfg_handle, *id); vdev->dev_no = *cfg_handle; + vdev->port_id = *id; } vdev->dev.parent = parent; @@ -289,25 +364,41 @@ static struct vio_dev *vio_create_one(struct mdesc_handle *hp, u64 mp, if (parent == NULL) { dp = cdev_node; } else if (to_vio_dev(parent) == root_vdev) { - dp = of_get_next_child(cdev_node, NULL); - while (dp) { - if (!strcmp(dp->type, type)) + for_each_child_of_node(cdev_node, dp) { + if (of_node_is_type(dp, type)) break; - - dp = of_get_next_child(cdev_node, dp); } } else { dp = to_vio_dev(parent)->dp; } vdev->dp = dp; - printk(KERN_INFO "VIO: Adding device %s\n", dev_name(&vdev->dev)); + /* + * node_name is NULL for the parent/channel-devices node and + * the parent doesn't require the MD node info. + */ + if (node_name != NULL) { + (void) snprintf(vdev->node_name, VIO_MAX_NAME_LEN, "%s", + node_name); + + err = mdesc_get_node_info(hp, mp, node_name, + &vdev->md_node_info); + if (err) { + pr_err("VIO: Could not get MD node info %s, err=%d\n", + dev_name(&vdev->dev), err); + kfree(vdev); + return NULL; + } + } + + pr_info("VIO: Adding device %s (tx_ino = %llx, rx_ino = %llx)\n", + dev_name(&vdev->dev), vdev->tx_ino, vdev->rx_ino); err = device_register(&vdev->dev); if (err) { printk(KERN_ERR "VIO: Could not register device %s, err=%d\n", dev_name(&vdev->dev), err); - kfree(vdev); + put_device(&vdev->dev); return NULL; } if (vdev->dp) @@ -317,32 +408,50 @@ static struct vio_dev *vio_create_one(struct mdesc_handle *hp, u64 mp, return vdev; } -static void vio_add(struct mdesc_handle *hp, u64 node) +static void vio_add(struct mdesc_handle *hp, u64 node, + const char *node_name) { - (void) vio_create_one(hp, node, &root_vdev->dev); + (void) vio_create_one(hp, node, node_name, &root_vdev->dev); } -static int vio_md_node_match(struct device *dev, void *arg) +struct vio_remove_node_data { + struct mdesc_handle *hp; + u64 node; +}; + +static int vio_md_node_match(struct device *dev, const void *arg) { struct vio_dev *vdev = to_vio_dev(dev); + const struct vio_remove_node_data *node_data; + u64 node; - if (vdev->mp == (u64) arg) - return 1; + node_data = (const struct vio_remove_node_data *)arg; - return 0; + node = vio_vdev_node(node_data->hp, vdev); + + if (node == node_data->node) + return 1; + else + return 0; } -static void vio_remove(struct mdesc_handle *hp, u64 node) +static void vio_remove(struct mdesc_handle *hp, u64 node, const char *node_name) { + struct vio_remove_node_data node_data; struct device *dev; - dev = device_find_child(&root_vdev->dev, (void *) node, + node_data.hp = hp; + node_data.node = node; + + dev = device_find_child(&root_vdev->dev, (void *)&node_data, vio_md_node_match); if (dev) { printk(KERN_INFO "VIO: Removing device %s\n", dev_name(dev)); device_unregister(dev); put_device(dev); + } else { + pr_err("VIO: %s node not found in MDESC\n", node_name); } } @@ -357,7 +466,8 @@ static struct mdesc_notifier_client vio_device_notifier = { * under "openboot" that we should not mess with as aparently that is * reserved exclusively for OBP use. */ -static void vio_add_ds(struct mdesc_handle *hp, u64 node) +static void vio_add_ds(struct mdesc_handle *hp, u64 node, + const char *node_name) { int found; u64 a; @@ -374,7 +484,7 @@ static void vio_add_ds(struct mdesc_handle *hp, u64 node) } if (found) - (void) vio_create_one(hp, node, &root_vdev->dev); + (void) vio_create_one(hp, node, node_name, &root_vdev->dev); } static struct mdesc_notifier_client vio_ds_notifier = { @@ -441,7 +551,7 @@ static int __init vio_init(void) cdev_cfg_handle = *cfg_handle; - root_vdev = vio_create_one(hp, root, NULL); + root_vdev = vio_create_one(hp, root, NULL, NULL); err = -ENODEV; if (!root_vdev) { printk(KERN_ERR "VIO: Could not create root device.\n"); diff --git a/arch/sparc/kernel/viohs.c b/arch/sparc/kernel/viohs.c index f8e7dd53e1c7..8fb2e7ca5015 100644 --- a/arch/sparc/kernel/viohs.c +++ b/arch/sparc/kernel/viohs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* viohs.c: LDOM Virtual I/O handshake helper layer. * * Copyright (C) 2007 David S. Miller <davem@davemloft.net> @@ -8,6 +9,7 @@ #include <linux/string.h> #include <linux/delay.h> #include <linux/sched.h> +#include <linux/sched/clock.h> #include <linux/slab.h> #include <asm/ldc.h> @@ -178,11 +180,17 @@ static int send_dreg(struct vio_driver_state *vio) struct vio_dring_register pkt; char all[sizeof(struct vio_dring_register) + (sizeof(struct ldc_trans_cookie) * - dr->ncookies)]; + VIO_MAX_RING_COOKIES)]; } u; + size_t bytes = sizeof(struct vio_dring_register) + + (sizeof(struct ldc_trans_cookie) * + dr->ncookies); int i; - memset(&u, 0, sizeof(u)); + if (WARN_ON(bytes > sizeof(u))) + return -EINVAL; + + memset(&u, 0, bytes); init_tag(&u.pkt.tag, VIO_TYPE_CTRL, VIO_SUBTYPE_INFO, VIO_DRING_REG); u.pkt.dring_ident = 0; u.pkt.num_descr = dr->num_entries; @@ -204,7 +212,7 @@ static int send_dreg(struct vio_driver_state *vio) (unsigned long long) u.pkt.cookies[i].cookie_size); } - return send_ctrl(vio, &u.pkt.tag, sizeof(u)); + return send_ctrl(vio, &u.pkt.tag, bytes); } static int send_rdx(struct vio_driver_state *vio) @@ -222,6 +230,9 @@ static int send_rdx(struct vio_driver_state *vio) static int send_attr(struct vio_driver_state *vio) { + if (!vio->ops) + return -EINVAL; + return vio->ops->send_attr(vio); } @@ -282,6 +293,7 @@ static int process_ver_info(struct vio_driver_state *vio, ver.minor = vap->minor; pkt->minor = ver.minor; pkt->tag.stype = VIO_SUBTYPE_ACK; + pkt->dev_class = vio->dev_class; viodbg(HS, "SEND VERSION ACK maj[%u] min[%u]\n", pkt->major, pkt->minor); err = send_ctrl(vio, &pkt->tag, sizeof(*pkt)); @@ -373,6 +385,9 @@ static int process_attr(struct vio_driver_state *vio, void *pkt) if (!(vio->hs_state & VIO_HS_GOTVERS)) return handshake_failure(vio); + if (!vio->ops) + return 0; + err = vio->ops->handle_attr(vio, pkt); if (err < 0) { return handshake_failure(vio); @@ -387,6 +402,7 @@ static int process_attr(struct vio_driver_state *vio, void *pkt) vio->hs_state |= VIO_HS_SENT_DREG; } } + return 0; } @@ -412,7 +428,7 @@ static int process_dreg_info(struct vio_driver_state *vio, struct vio_dring_register *pkt) { struct vio_dring_state *dr; - int i, len; + int i; viodbg(HS, "GOT DRING_REG INFO ident[%llx] " "ndesc[%u] dsz[%u] opt[0x%x] ncookies[%u]\n", @@ -426,6 +442,13 @@ static int process_dreg_info(struct vio_driver_state *vio, if (vio->dr_state & VIO_DR_STATE_RXREG) goto send_nack; + /* v1.6 and higher, ACK with desired, supported mode, or NACK */ + if (vio_version_after_eq(vio, 1, 6)) { + if (!(pkt->options & VIO_TX_DRING)) + goto send_nack; + pkt->options = VIO_TX_DRING; + } + BUG_ON(vio->desc_buf); vio->desc_buf = kzalloc(pkt->descr_size, GFP_ATOMIC); @@ -453,12 +476,13 @@ static int process_dreg_info(struct vio_driver_state *vio, pkt->tag.stype = VIO_SUBTYPE_ACK; pkt->dring_ident = ++dr->ident; - viodbg(HS, "SEND DRING_REG ACK ident[%llx]\n", - (unsigned long long) pkt->dring_ident); + viodbg(HS, "SEND DRING_REG ACK ident[%llx] " + "ndesc[%u] dsz[%u] opt[0x%x] ncookies[%u]\n", + (unsigned long long) pkt->dring_ident, + pkt->num_descr, pkt->descr_size, pkt->options, + pkt->num_cookies); - len = (sizeof(*pkt) + - (dr->ncookies * sizeof(struct ldc_trans_cookie))); - if (send_ctrl(vio, &pkt->tag, len) < 0) + if (send_ctrl(vio, &pkt->tag, struct_size(pkt, cookies, dr->ncookies)) < 0) goto send_nack; vio->dr_state |= VIO_DR_STATE_RXREG; @@ -636,10 +660,13 @@ int vio_control_pkt_engine(struct vio_driver_state *vio, void *pkt) err = process_unknown(vio, pkt); break; } + if (!err && vio->hs_state != prev_state && - (vio->hs_state & VIO_HS_COMPLETE)) - vio->ops->handshake_complete(vio); + (vio->hs_state & VIO_HS_COMPLETE)) { + if (vio->ops) + vio->ops->handshake_complete(vio); + } return err; } @@ -714,7 +741,7 @@ int vio_ldc_alloc(struct vio_driver_state *vio, cfg.tx_irq = vio->vdev->tx_irq; cfg.rx_irq = vio->vdev->rx_irq; - lp = ldc_alloc(vio->vdev->channel_id, &cfg, event_arg); + lp = ldc_alloc(vio->vdev->channel_id, &cfg, event_arg, vio->name); if (IS_ERR(lp)) return PTR_ERR(lp); @@ -746,7 +773,7 @@ void vio_port_up(struct vio_driver_state *vio) err = 0; if (state == LDC_STATE_INIT) { - err = ldc_bind(vio->lp, vio->name); + err = ldc_bind(vio->lp); if (err) printk(KERN_WARNING "%s: Port %lu bind failed, " "err=%d\n", @@ -754,7 +781,11 @@ void vio_port_up(struct vio_driver_state *vio) } if (!err) { - err = ldc_connect(vio->lp); + if (ldc_mode(vio->lp) == LDC_MODE_RAW) + ldc_set_state(vio->lp, LDC_STATE_CONNECTED); + else + err = ldc_connect(vio->lp); + if (err) printk(KERN_WARNING "%s: Port %lu connect failed, " "err=%d\n", @@ -771,9 +802,9 @@ void vio_port_up(struct vio_driver_state *vio) } EXPORT_SYMBOL(vio_port_up); -static void vio_port_timer(unsigned long _arg) +static void vio_port_timer(struct timer_list *t) { - struct vio_driver_state *vio = (struct vio_driver_state *) _arg; + struct vio_driver_state *vio = timer_container_of(vio, t, timer); vio_port_up(vio); } @@ -788,16 +819,21 @@ int vio_driver_init(struct vio_driver_state *vio, struct vio_dev *vdev, case VDEV_NETWORK_SWITCH: case VDEV_DISK: case VDEV_DISK_SERVER: + case VDEV_CONSOLE_CON: break; default: return -EINVAL; } - if (!ops->send_attr || - !ops->handle_attr || - !ops->handshake_complete) - return -EINVAL; + if (dev_class == VDEV_NETWORK || + dev_class == VDEV_NETWORK_SWITCH || + dev_class == VDEV_DISK || + dev_class == VDEV_DISK_SERVER) { + if (!ops || !ops->send_attr || !ops->handle_attr || + !ops->handshake_complete) + return -EINVAL; + } if (!ver_table || ver_table_size < 0) return -EINVAL; @@ -817,7 +853,7 @@ int vio_driver_init(struct vio_driver_state *vio, struct vio_dev *vdev, vio->ops = ops; - setup_timer(&vio->timer, vio_port_timer, (unsigned long) vio); + timer_setup(&vio->timer, vio_port_timer, 0); return 0; } diff --git a/arch/sparc/kernel/visemul.c b/arch/sparc/kernel/visemul.c index c096c624ac4d..64ed80ed6cc2 100644 --- a/arch/sparc/kernel/visemul.c +++ b/arch/sparc/kernel/visemul.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* visemul.c: Emulation of VIS instructions. * * Copyright (C) 2006 David S. Miller (davem@davemloft.net) @@ -10,7 +11,7 @@ #include <asm/ptrace.h> #include <asm/pstate.h> #include <asm/fpumacro.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/cacheflush.h> /* OPF field of various VIS instructions. */ @@ -30,7 +31,7 @@ /* 001001011 - two 32-bit merges */ #define FPMERGE_OPF 0x04b -/* 000110001 - 8-by-16-bit partitoned product */ +/* 000110001 - 8-by-16-bit partitioned product */ #define FMUL8x16_OPF 0x031 /* 000110011 - 8-by-16-bit upper alpha partitioned product */ diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index 0bacceb19150..f1b86eb30340 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* ld script for sparc32/sparc64 kernel */ #include <asm-generic/vmlinux.lds.h> @@ -33,20 +34,31 @@ ENTRY(_start) jiffies = jiffies_64; #endif +#ifdef CONFIG_SPARC64 +ASSERT((swapper_tsb == 0x0000000000408000), "Error: sparc64 early assembler too large") +#endif + SECTIONS { - /* swapper_low_pmd_dir is sparc64 only */ - swapper_low_pmd_dir = 0x0000000000402000; +#ifdef CONFIG_SPARC64 + swapper_pg_dir = 0x0000000000402000; +#endif . = INITIAL_ADDRESS; .text TEXTSTART : { _text = .; HEAD_TEXT + ALIGN_FUNCTION(); +#ifdef CONFIG_SPARC64 + /* Match text section symbols in head_64.S first */ + *head_64.o(.text) +#endif TEXT_TEXT SCHED_TEXT LOCK_TEXT KPROBES_TEXT IRQENTRY_TEXT + SOFTIRQENTRY_TEXT *(.gnu.warning) } = 0 _etext = .; @@ -59,7 +71,7 @@ SECTIONS .data1 : { *(.data1) } - RW_DATA_SECTION(SMP_CACHE_BYTES, 0, THREAD_SIZE) + RW_DATA(SMP_CACHE_BYTES, 0, THREAD_SIZE) /* End of data section */ _edata = .; @@ -70,7 +82,6 @@ SECTIONS __stop___fixup = .; } EXCEPTION_TABLE(16) - NOTES . = ALIGN(PAGE_SIZE); __init_begin = ALIGN(PAGE_SIZE); @@ -137,15 +148,50 @@ SECTIONS *(.pause_3insn_patch) __pause_3insn_patch_end = .; } + .sun_m7_1insn_patch : { + __sun_m7_1insn_patch = .; + *(.sun_m7_1insn_patch) + __sun_m7_1insn_patch_end = .; + } + .sun_m7_2insn_patch : { + __sun_m7_2insn_patch = .; + *(.sun_m7_2insn_patch) + __sun_m7_2insn_patch_end = .; + } + .get_tick_patch : { + __get_tick_patch = .; + *(.get_tick_patch) + __get_tick_patch_end = .; + } + .pud_huge_patch : { + __pud_huge_patch = .; + *(.pud_huge_patch) + __pud_huge_patch_end = .; + } + .fast_win_ctrl_1insn_patch : { + __fast_win_ctrl_1insn_patch = .; + *(.fast_win_ctrl_1insn_patch) + __fast_win_ctrl_1insn_patch_end = .; + } PERCPU_SECTION(SMP_CACHE_BYTES) . = ALIGN(PAGE_SIZE); + .exit.text : { + EXIT_TEXT + } + + .exit.data : { + EXIT_DATA + } + + . = ALIGN(PAGE_SIZE); __init_end = .; BSS_SECTION(0, 0, 0) _end = . ; STABS_DEBUG DWARF_DEBUG + ELF_DETAILS DISCARDS } diff --git a/arch/sparc/kernel/windows.c b/arch/sparc/kernel/windows.c index 3107381e576d..8f20862ccc83 100644 --- a/arch/sparc/kernel/windows.c +++ b/arch/sparc/kernel/windows.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* windows.c: Routines to deal with register window management * at the C-code level. * @@ -10,7 +11,10 @@ #include <linux/mm.h> #include <linux/smp.h> -#include <asm/uaccess.h> +#include <asm/cacheflush.h> +#include <linux/uaccess.h> + +#include "kernel.h" /* Do save's until all user register windows are out of the cpu. */ void flush_user_windows(void) @@ -117,8 +121,10 @@ void try_to_clear_window_buffer(struct pt_regs *regs, int who) if ((sp & 7) || copy_to_user((char __user *) sp, &tp->reg_window[window], - sizeof(struct reg_window32))) - do_exit(SIGILL); + sizeof(struct reg_window32))) { + force_exit_sig(SIGILL); + return; + } } tp->w_saved = 0; } diff --git a/arch/sparc/kernel/winfixup.S b/arch/sparc/kernel/winfixup.S index 1e67ce958369..448accee090f 100644 --- a/arch/sparc/kernel/winfixup.S +++ b/arch/sparc/kernel/winfixup.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* winfixup.S: Handle cases where user stack pointer is found to be bogus. * * Copyright (C) 1997, 2006 David S. Miller (davem@davemloft.net) @@ -32,8 +33,7 @@ fill_fixup: rd %pc, %g7 call do_sparc64_fault add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop + ba,a,pt %xcc, rtrap /* Be very careful about usage of the trap globals here. * You cannot touch %g5 as that has the fault information. @@ -153,6 +153,8 @@ fill_fixup_dax: call sun4v_data_access_exception nop ba,a,pt %xcc, rtrap + nop 1: call spitfire_data_access_exception nop ba,a,pt %xcc, rtrap + nop diff --git a/arch/sparc/kernel/wof.S b/arch/sparc/kernel/wof.S index 28a7bc69f82b..96a3a112423a 100644 --- a/arch/sparc/kernel/wof.S +++ b/arch/sparc/kernel/wof.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * wof.S: Sparc window overflow handler. * diff --git a/arch/sparc/kernel/wuf.S b/arch/sparc/kernel/wuf.S index 2c21cc59683e..1a4ca490e9c2 100644 --- a/arch/sparc/kernel/wuf.S +++ b/arch/sparc/kernel/wuf.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * wuf.S: Window underflow trap handler for the Sparc. * diff --git a/arch/sparc/lib/COPYING.LIB b/arch/sparc/lib/COPYING.LIB deleted file mode 100644 index eb685a5ec981..000000000000 --- a/arch/sparc/lib/COPYING.LIB +++ /dev/null @@ -1,481 +0,0 @@ - GNU LIBRARY GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1991 Free Software Foundation, Inc. - 675 Mass Ave, Cambridge, MA 02139, USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - -[This is the first released version of the library GPL. It is - numbered 2 because it goes with version 2 of the ordinary GPL.] - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -Licenses are intended to guarantee your freedom to share and change -free software--to make sure the software is free for all its users. - - This license, the Library General Public License, applies to some -specially designated Free Software Foundation software, and to any -other libraries whose authors decide to use it. You can use it for -your libraries, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if -you distribute copies of the library, or if you modify it. - - For example, if you distribute copies of the library, whether gratis -or for a fee, you must give the recipients all the rights that we gave -you. You must make sure that they, too, receive or can get the source -code. If you link a program with the library, you must provide -complete object files to the recipients so that they can relink them -with the library, after making changes to the library and recompiling -it. And you must show them these terms so they know their rights. - - Our method of protecting your rights has two steps: (1) copyright -the library, and (2) offer you this license which gives you legal -permission to copy, distribute and/or modify the library. - - Also, for each distributor's protection, we want to make certain -that everyone understands that there is no warranty for this free -library. If the library is modified by someone else and passed on, we -want its recipients to know that what they have is not the original -version, so that any problems introduced by others will not reflect on -the original authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that companies distributing free -software will individually obtain patent licenses, thus in effect -transforming the program into proprietary software. To prevent this, -we have made it clear that any patent must be licensed for everyone's -free use or not licensed at all. - - Most GNU software, including some libraries, is covered by the ordinary -GNU General Public License, which was designed for utility programs. This -license, the GNU Library General Public License, applies to certain -designated libraries. This license is quite different from the ordinary -one; be sure to read it in full, and don't assume that anything in it is -the same as in the ordinary license. - - The reason we have a separate public license for some libraries is that -they blur the distinction we usually make between modifying or adding to a -program and simply using it. Linking a program with a library, without -changing the library, is in some sense simply using the library, and is -analogous to running a utility program or application program. However, in -a textual and legal sense, the linked executable is a combined work, a -derivative of the original library, and the ordinary General Public License -treats it as such. - - Because of this blurred distinction, using the ordinary General -Public License for libraries did not effectively promote software -sharing, because most developers did not use the libraries. We -concluded that weaker conditions might promote sharing better. - - However, unrestricted linking of non-free programs would deprive the -users of those programs of all benefit from the free status of the -libraries themselves. This Library General Public License is intended to -permit developers of non-free programs to use free libraries, while -preserving your freedom as a user of such programs to change the free -libraries that are incorporated in them. (We have not seen how to achieve -this as regards changes in header files, but we have achieved it as regards -changes in the actual functions of the Library.) The hope is that this -will lead to faster development of free libraries. - - The precise terms and conditions for copying, distribution and -modification follow. Pay close attention to the difference between a -"work based on the library" and a "work that uses the library". The -former contains code derived from the library, while the latter only -works together with the library. - - Note that it is possible for a library to be covered by the ordinary -General Public License rather than by this special one. - - GNU LIBRARY GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License Agreement applies to any software library which -contains a notice placed by the copyright holder or other authorized -party saying it may be distributed under the terms of this Library -General Public License (also called "this License"). Each licensee is -addressed as "you". - - A "library" means a collection of software functions and/or data -prepared so as to be conveniently linked with application programs -(which use some of those functions and data) to form executables. - - The "Library", below, refers to any such software library or work -which has been distributed under these terms. A "work based on the -Library" means either the Library or any derivative work under -copyright law: that is to say, a work containing the Library or a -portion of it, either verbatim or with modifications and/or translated -straightforwardly into another language. (Hereinafter, translation is -included without limitation in the term "modification".) - - "Source code" for a work means the preferred form of the work for -making modifications to it. For a library, complete source code means -all the source code for all modules it contains, plus any associated -interface definition files, plus the scripts used to control compilation -and installation of the library. - - Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running a program using the Library is not restricted, and output from -such a program is covered only if its contents constitute a work based -on the Library (independent of the use of the Library in a tool for -writing it). Whether that is true depends on what the Library does -and what the program that uses the Library does. - - 1. You may copy and distribute verbatim copies of the Library's -complete source code as you receive it, in any medium, provided that -you conspicuously and appropriately publish on each copy an -appropriate copyright notice and disclaimer of warranty; keep intact -all the notices that refer to this License and to the absence of any -warranty; and distribute a copy of this License along with the -Library. - - You may charge a fee for the physical act of transferring a copy, -and you may at your option offer warranty protection in exchange for a -fee. - - 2. You may modify your copy or copies of the Library or any portion -of it, thus forming a work based on the Library, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) The modified work must itself be a software library. - - b) You must cause the files modified to carry prominent notices - stating that you changed the files and the date of any change. - - c) You must cause the whole of the work to be licensed at no - charge to all third parties under the terms of this License. - - d) If a facility in the modified Library refers to a function or a - table of data to be supplied by an application program that uses - the facility, other than as an argument passed when the facility - is invoked, then you must make a good faith effort to ensure that, - in the event an application does not supply such function or - table, the facility still operates, and performs whatever part of - its purpose remains meaningful. - - (For example, a function in a library to compute square roots has - a purpose that is entirely well-defined independent of the - application. Therefore, Subsection 2d requires that any - application-supplied function or table used by this function must - be optional: if the application does not supply it, the square - root function must still compute square roots.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Library, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Library, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote -it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Library. - -In addition, mere aggregation of another work not based on the Library -with the Library (or with a work based on the Library) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may opt to apply the terms of the ordinary GNU General Public -License instead of this License to a given copy of the Library. To do -this, you must alter all the notices that refer to this License, so -that they refer to the ordinary GNU General Public License, version 2, -instead of to this License. (If a newer version than version 2 of the -ordinary GNU General Public License has appeared, then you can specify -that version instead if you wish.) Do not make any other change in -these notices. - - Once this change is made in a given copy, it is irreversible for -that copy, so the ordinary GNU General Public License applies to all -subsequent copies and derivative works made from that copy. - - This option is useful when you wish to copy part of the code of -the Library into a program that is not a library. - - 4. You may copy and distribute the Library (or a portion or -derivative of it, under Section 2) in object code or executable form -under the terms of Sections 1 and 2 above provided that you accompany -it with the complete corresponding machine-readable source code, which -must be distributed under the terms of Sections 1 and 2 above on a -medium customarily used for software interchange. - - If distribution of object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the -source code from the same place satisfies the requirement to -distribute the source code, even though third parties are not -compelled to copy the source along with the object code. - - 5. A program that contains no derivative of any portion of the -Library, but is designed to work with the Library by being compiled or -linked with it, is called a "work that uses the Library". Such a -work, in isolation, is not a derivative work of the Library, and -therefore falls outside the scope of this License. - - However, linking a "work that uses the Library" with the Library -creates an executable that is a derivative of the Library (because it -contains portions of the Library), rather than a "work that uses the -library". The executable is therefore covered by this License. -Section 6 states terms for distribution of such executables. - - When a "work that uses the Library" uses material from a header file -that is part of the Library, the object code for the work may be a -derivative work of the Library even though the source code is not. -Whether this is true is especially significant if the work can be -linked without the Library, or if the work is itself a library. The -threshold for this to be true is not precisely defined by law. - - If such an object file uses only numerical parameters, data -structure layouts and accessors, and small macros and small inline -functions (ten lines or less in length), then the use of the object -file is unrestricted, regardless of whether it is legally a derivative -work. (Executables containing this object code plus portions of the -Library will still fall under Section 6.) - - Otherwise, if the work is a derivative of the Library, you may -distribute the object code for the work under the terms of Section 6. -Any executables containing that work also fall under Section 6, -whether or not they are linked directly with the Library itself. - - 6. As an exception to the Sections above, you may also compile or -link a "work that uses the Library" with the Library to produce a -work containing portions of the Library, and distribute that work -under terms of your choice, provided that the terms permit -modification of the work for the customer's own use and reverse -engineering for debugging such modifications. - - You must give prominent notice with each copy of the work that the -Library is used in it and that the Library and its use are covered by -this License. You must supply a copy of this License. If the work -during execution displays copyright notices, you must include the -copyright notice for the Library among them, as well as a reference -directing the user to the copy of this License. Also, you must do one -of these things: - - a) Accompany the work with the complete corresponding - machine-readable source code for the Library including whatever - changes were used in the work (which must be distributed under - Sections 1 and 2 above); and, if the work is an executable linked - with the Library, with the complete machine-readable "work that - uses the Library", as object code and/or source code, so that the - user can modify the Library and then relink to produce a modified - executable containing the modified Library. (It is understood - that the user who changes the contents of definitions files in the - Library will not necessarily be able to recompile the application - to use the modified definitions.) - - b) Accompany the work with a written offer, valid for at - least three years, to give the same user the materials - specified in Subsection 6a, above, for a charge no more - than the cost of performing this distribution. - - c) If distribution of the work is made by offering access to copy - from a designated place, offer equivalent access to copy the above - specified materials from the same place. - - d) Verify that the user has already received a copy of these - materials or that you have already sent this user a copy. - - For an executable, the required form of the "work that uses the -Library" must include any data and utility programs needed for -reproducing the executable from it. However, as a special exception, -the source code distributed need not include anything that is normally -distributed (in either source or binary form) with the major -components (compiler, kernel, and so on) of the operating system on -which the executable runs, unless that component itself accompanies -the executable. - - It may happen that this requirement contradicts the license -restrictions of other proprietary libraries that do not normally -accompany the operating system. Such a contradiction means you cannot -use both them and the Library together in an executable that you -distribute. - - 7. You may place library facilities that are a work based on the -Library side-by-side in a single library together with other library -facilities not covered by this License, and distribute such a combined -library, provided that the separate distribution of the work based on -the Library and of the other library facilities is otherwise -permitted, and provided that you do these two things: - - a) Accompany the combined library with a copy of the same work - based on the Library, uncombined with any other library - facilities. This must be distributed under the terms of the - Sections above. - - b) Give prominent notice with the combined library of the fact - that part of it is a work based on the Library, and explaining - where to find the accompanying uncombined form of the same work. - - 8. You may not copy, modify, sublicense, link with, or distribute -the Library except as expressly provided under this License. Any -attempt otherwise to copy, modify, sublicense, link with, or -distribute the Library is void, and will automatically terminate your -rights under this License. However, parties who have received copies, -or rights, from you under this License will not have their licenses -terminated so long as such parties remain in full compliance. - - 9. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Library or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Library (or any work based on the -Library), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Library or works based on it. - - 10. Each time you redistribute the Library (or any work based on the -Library), the recipient automatically receives a license from the -original licensor to copy, distribute, link with or modify the Library -subject to these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 11. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Library at all. For example, if a patent -license would not permit royalty-free redistribution of the Library by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Library. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply, -and the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 12. If the distribution and/or use of the Library is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Library under this License may add -an explicit geographical distribution limitation excluding those countries, -so that distribution is permitted only in or among countries not thus -excluded. In such case, this License incorporates the limitation as if -written in the body of this License. - - 13. The Free Software Foundation may publish revised and/or new -versions of the Library General Public License from time to time. -Such new versions will be similar in spirit to the present version, -but may differ in detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Library -specifies a version number of this License which applies to it and -"any later version", you have the option of following the terms and -conditions either of that version or of any later version published by -the Free Software Foundation. If the Library does not specify a -license version number, you may choose any version ever published by -the Free Software Foundation. - - 14. If you wish to incorporate parts of the Library into other free -programs whose distribution conditions are incompatible with these, -write to the author to ask for permission. For software which is -copyrighted by the Free Software Foundation, write to the Free -Software Foundation; we sometimes make exceptions for this. Our -decision will be guided by the two goals of preserving the free status -of all derivatives of our free software and of promoting the sharing -and reuse of software generally. - - NO WARRANTY - - 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO -WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. -EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR -OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY -KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE -LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME -THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN -WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY -AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU -FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR -CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE -LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING -RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF -SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH -DAMAGES. - - END OF TERMS AND CONDITIONS - - Appendix: How to Apply These Terms to Your New Libraries - - If you develop a new library, and you want it to be of the greatest -possible use to the public, we recommend making it free software that -everyone can redistribute and change. You can do so by permitting -redistribution under these terms (or, alternatively, under the terms of the -ordinary General Public License). - - To apply these terms, attach the following notices to the library. It is -safest to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least the -"copyright" line and a pointer to where the full notice is found. - - <one line to give the library's name and a brief idea of what it does.> - Copyright (C) <year> <name of author> - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Library General Public - License as published by the Free Software Foundation; either - version 2 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Library General Public License for more details. - - You should have received a copy of the GNU Library General Public - License along with this library; if not, write to the Free - Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -Also add information on how to contact you by electronic and paper mail. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the library, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the - library `Frob' (a library for tweaking knobs) written by James Random Hacker. - - <signature of Ty Coon>, 1 April 1990 - Ty Coon, President of Vice - -That's all there is to it! diff --git a/arch/sparc/lib/GENbzero.S b/arch/sparc/lib/GENbzero.S index 8e7a843ddd88..63d618857d49 100644 --- a/arch/sparc/lib/GENbzero.S +++ b/arch/sparc/lib/GENbzero.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* GENbzero.S: Generic sparc64 memset/clear_user. * * Copyright (C) 2007 David S. Miller (davem@davemloft.net) @@ -8,7 +9,7 @@ 98: x,y; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_o1; \ + .word 98b, __retl_o1_asi;\ .text; \ .align 4; diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S index b7d0bd6b1406..6891a5678ea3 100644 --- a/arch/sparc/lib/GENcopy_from_user.S +++ b/arch/sparc/lib/GENcopy_from_user.S @@ -1,13 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* GENcopy_from_user.S: Generic sparc64 copy from userspace. * * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; @@ -23,7 +24,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S index 780550e1afc7..df75b532a934 100644 --- a/arch/sparc/lib/GENcopy_to_user.S +++ b/arch/sparc/lib/GENcopy_to_user.S @@ -1,13 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* GENcopy_to_user.S: Generic sparc64 copy to userspace. * * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; @@ -27,7 +28,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/GENmemcpy.S b/arch/sparc/lib/GENmemcpy.S index 89358ee94851..114340a0d36e 100644 --- a/arch/sparc/lib/GENmemcpy.S +++ b/arch/sparc/lib/GENmemcpy.S @@ -1,24 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* GENmemcpy.S: Generic sparc64 memcpy. * * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ #ifdef __KERNEL__ +#include <linux/linkage.h> #define GLOBAL_SPARE %g7 #else #define GLOBAL_SPARE %g5 #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x -#endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#define EX_ST(x,y) x #endif #ifndef LOAD @@ -45,6 +43,29 @@ .register %g3,#scratch .text + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +ENTRY(GEN_retl_o4_1) + add %o4, %o2, %o4 + retl + add %o4, 1, %o0 +ENDPROC(GEN_retl_o4_1) +ENTRY(GEN_retl_g1_8) + add %g1, %o2, %g1 + retl + add %g1, 8, %o0 +ENDPROC(GEN_retl_g1_8) +ENTRY(GEN_retl_o2_4) + retl + add %o2, 4, %o0 +ENDPROC(GEN_retl_o2_4) +ENTRY(GEN_retl_o2_1) + retl + add %o2, 1, %o0 +ENDPROC(GEN_retl_o2_1) +#endif + .align 64 .globl FUNC_NAME @@ -73,8 +94,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %g0, %o4, %o4 sub %o2, %o4, %o2 1: subcc %o4, 1, %o4 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o0)) + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o4_1) + EX_ST(STORE(stb, %g1, %o0),GEN_retl_o4_1) add %o1, 1, %o1 bne,pt %XCC, 1b add %o0, 1, %o0 @@ -82,8 +103,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0x7, %g1 sub %o2, %g1, %o2 1: subcc %g1, 0x8, %g1 - EX_LD(LOAD(ldx, %o1, %g2)) - EX_ST(STORE(stx, %g2, %o0)) + EX_LD(LOAD(ldx, %o1, %g2),GEN_retl_g1_8) + EX_ST(STORE(stx, %g2, %o0),GEN_retl_g1_8) add %o1, 0x8, %o1 bne,pt %XCC, 1b add %o0, 0x8, %o0 @@ -100,8 +121,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 1: subcc %o2, 4, %o2 - EX_LD(LOAD(lduw, %o1, %g1)) - EX_ST(STORE(stw, %g1, %o1 + %o3)) + EX_LD(LOAD(lduw, %o1, %g1),GEN_retl_o2_4) + EX_ST(STORE(stw, %g1, %o1 + %o3),GEN_retl_o2_4) bgu,pt %XCC, 1b add %o1, 4, %o1 @@ -111,8 +132,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 32 90: subcc %o2, 1, %o2 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o1 + %o3)) + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o2_1) + EX_ST(STORE(stb, %g1, %o1 + %o3),GEN_retl_o2_1) bgu,pt %XCC, 90b add %o1, 1, %o1 retl diff --git a/arch/sparc/lib/GENpage.S b/arch/sparc/lib/GENpage.S index 2ef9d05f21bc..c143c4d1de3f 100644 --- a/arch/sparc/lib/GENpage.S +++ b/arch/sparc/lib/GENpage.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* GENpage.S: Generic clear and copy page. * * Copyright (C) 2007 (davem@davemloft.net) diff --git a/arch/sparc/lib/GENpatch.S b/arch/sparc/lib/GENpatch.S index fab9e89f16bd..1ec1f02c8b7b 100644 --- a/arch/sparc/lib/GENpatch.S +++ b/arch/sparc/lib/GENpatch.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* GENpatch.S: Patch Ultra-I routines with generic variant. * * Copyright (C) 2007 David S. Miller <davem@davemloft.net> @@ -26,8 +27,8 @@ .type generic_patch_copyops,#function generic_patch_copyops: GEN_DO_PATCH(memcpy, GENmemcpy) - GEN_DO_PATCH(___copy_from_user, GENcopy_from_user) - GEN_DO_PATCH(___copy_to_user, GENcopy_to_user) + GEN_DO_PATCH(raw_copy_from_user, GENcopy_from_user) + GEN_DO_PATCH(raw_copy_to_user, GENcopy_to_user) retl nop .size generic_patch_copyops,.-generic_patch_copyops diff --git a/arch/sparc/lib/M7copy_from_user.S b/arch/sparc/lib/M7copy_from_user.S new file mode 100644 index 000000000000..66464b3e3649 --- /dev/null +++ b/arch/sparc/lib/M7copy_from_user.S @@ -0,0 +1,40 @@ +/* + * M7copy_from_user.S: SPARC M7 optimized copy from userspace. + * + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + */ + + +#define EX_LD(x, y) \ +98: x; \ + .section __ex_table,"a"; \ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_LD_FP(x, y) \ +98: x; \ + .section __ex_table,"a"; \ + .align 4; \ + .word 98b, y##_fp; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#define FUNC_NAME M7copy_from_user +#define LOAD(type,addr,dest) type##a [addr] %asi, dest +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop +#endif + +#include "M7memcpy.S" diff --git a/arch/sparc/lib/M7copy_to_user.S b/arch/sparc/lib/M7copy_to_user.S new file mode 100644 index 000000000000..a60ac467f808 --- /dev/null +++ b/arch/sparc/lib/M7copy_to_user.S @@ -0,0 +1,51 @@ +/* + * M7copy_to_user.S: SPARC M7 optimized copy to userspace. + * + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + */ + + +#define EX_ST(x, y) \ +98: x; \ + .section __ex_table,"a"; \ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_ST_FP(x, y) \ +98: x; \ + .section __ex_table,"a"; \ + .align 4; \ + .word 98b, y##_fp; \ + .text; \ + .align 4; + + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#ifndef ASI_BLK_INIT_QUAD_LDD_AIUS +#define ASI_BLK_INIT_QUAD_LDD_AIUS 0x23 +#endif + +#define FUNC_NAME M7copy_to_user +#define STORE(type,src,addr) type##a src, [addr] %asi +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_AIUS +#define STORE_MRU_ASI ASI_ST_BLKINIT_MRU_S +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop +#endif + +#include "M7memcpy.S" diff --git a/arch/sparc/lib/M7memcpy.S b/arch/sparc/lib/M7memcpy.S new file mode 100644 index 000000000000..99357bfa8e82 --- /dev/null +++ b/arch/sparc/lib/M7memcpy.S @@ -0,0 +1,923 @@ +/* + * M7memcpy: Optimized SPARC M7 memcpy + * + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + */ + + .file "M7memcpy.S" + +/* + * memcpy(s1, s2, len) + * + * Copy s2 to s1, always copy n bytes. + * Note: this C code does not work for overlapped copies. + * + * Fast assembler language version of the following C-program for memcpy + * which represents the `standard' for the C-library. + * + * void * + * memcpy(void *s, const void *s0, size_t n) + * { + * if (n != 0) { + * char *s1 = s; + * const char *s2 = s0; + * do { + * *s1++ = *s2++; + * } while (--n != 0); + * } + * return (s); + * } + * + * + * SPARC T7/M7 Flow : + * + * if (count < SMALL_MAX) { + * if count < SHORTCOPY (SHORTCOPY=3) + * copy bytes; exit with dst addr + * if src & dst aligned on word boundary but not long word boundary, + * copy with ldw/stw; branch to finish_up + * if src & dst aligned on long word boundary + * copy with ldx/stx; branch to finish_up + * if src & dst not aligned and length <= SHORTCHECK (SHORTCHECK=14) + * copy bytes; exit with dst addr + * move enough bytes to get src to word boundary + * if dst now on word boundary + * move_words: + * copy words; branch to finish_up + * if dst now on half word boundary + * load words, shift half words, store words; branch to finish_up + * if dst on byte 1 + * load words, shift 3 bytes, store words; branch to finish_up + * if dst on byte 3 + * load words, shift 1 byte, store words; branch to finish_up + * finish_up: + * copy bytes; exit with dst addr + * } else { More than SMALL_MAX bytes + * move bytes until dst is on long word boundary + * if( src is on long word boundary ) { + * if (count < MED_MAX) { + * finish_long: src/dst aligned on 8 bytes + * copy with ldx/stx in 8-way unrolled loop; + * copy final 0-63 bytes; exit with dst addr + * } else { src/dst aligned; count > MED_MAX + * align dst on 64 byte boundary; for main data movement: + * prefetch src data to L2 cache; let HW prefetch move data to L1 cache + * Use BIS (block initializing store) to avoid copying store cache + * lines from memory. But pre-store first element of each cache line + * ST_CHUNK lines in advance of the rest of that cache line. That + * gives time for replacement cache lines to be written back without + * excess STQ and Miss Buffer filling. Repeat until near the end, + * then finish up storing before going to finish_long. + * } + * } else { src/dst not aligned on 8 bytes + * if src is word aligned and count < MED_WMAX + * move words in 8-way unrolled loop + * move final 0-31 bytes; exit with dst addr + * if count < MED_UMAX + * use alignaddr/faligndata combined with ldd/std in 8-way + * unrolled loop to move data. + * go to unalign_done + * else + * setup alignaddr for faligndata instructions + * align dst on 64 byte boundary; prefetch src data to L1 cache + * loadx8, falign, block-store, prefetch loop + * (only use block-init-store when src/dst on 8 byte boundaries.) + * unalign_done: + * move remaining bytes for unaligned cases. exit with dst addr. + * } + * + */ + +#include <asm/visasm.h> +#include <asm/asi.h> + +#if !defined(EX_LD) && !defined(EX_ST) +#define NON_USER_COPY +#endif + +#ifndef EX_LD +#define EX_LD(x,y) x +#endif +#ifndef EX_LD_FP +#define EX_LD_FP(x,y) x +#endif + +#ifndef EX_ST +#define EX_ST(x,y) x +#endif +#ifndef EX_ST_FP +#define EX_ST_FP(x,y) x +#endif + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +#endif + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef STORE +#define STORE(type,src,addr) type src, [addr] +#endif + +/* + * ASI_BLK_INIT_QUAD_LDD_P/ASI_BLK_INIT_QUAD_LDD_S marks the cache + * line as "least recently used" which means if many threads are + * active, it has a high probability of being pushed out of the cache + * between the first initializing store and the final stores. + * Thus, we use ASI_ST_BLKINIT_MRU_P/ASI_ST_BLKINIT_MRU_S which + * marks the cache line as "most recently used" for all + * but the last cache line + */ +#ifndef STORE_ASI +#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P +#else +#define STORE_ASI 0x80 /* ASI_P */ +#endif +#endif + +#ifndef STORE_MRU_ASI +#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA +#define STORE_MRU_ASI ASI_ST_BLKINIT_MRU_P +#else +#define STORE_MRU_ASI 0x80 /* ASI_P */ +#endif +#endif + +#ifndef STORE_INIT +#define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI +#endif + +#ifndef STORE_INIT_MRU +#define STORE_INIT_MRU(src,addr) stxa src, [addr] STORE_MRU_ASI +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME M7memcpy +#endif + +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#define BLOCK_SIZE 64 +#define SHORTCOPY 3 +#define SHORTCHECK 14 +#define SHORT_LONG 64 /* max copy for short longword-aligned case */ + /* must be at least 64 */ +#define SMALL_MAX 128 +#define MED_UMAX 1024 /* max copy for medium un-aligned case */ +#define MED_WMAX 1024 /* max copy for medium word-aligned case */ +#define MED_MAX 1024 /* max copy for medium longword-aligned case */ +#define ST_CHUNK 24 /* ST_CHUNK - block of values for BIS Store */ +#define ALIGN_PRE 24 /* distance for aligned prefetch loop */ + + .register %g2,#scratch + + .section ".text" + .global FUNC_NAME + .type FUNC_NAME, #function + .align 16 +FUNC_NAME: + srlx %o2, 31, %g2 + cmp %g2, 0 + tne %xcc, 5 + PREAMBLE + mov %o0, %g1 ! save %o0 + brz,pn %o2, .Lsmallx + cmp %o2, 3 + ble,pn %icc, .Ltiny_cp + cmp %o2, 19 + ble,pn %icc, .Lsmall_cp + or %o0, %o1, %g2 + cmp %o2, SMALL_MAX + bl,pn %icc, .Lmedium_cp + nop + +.Lmedium: + neg %o0, %o5 + andcc %o5, 7, %o5 ! bytes till DST 8 byte aligned + brz,pt %o5, .Ldst_aligned_on_8 + + ! %o5 has the bytes to be written in partial store. + sub %o2, %o5, %o2 + sub %o1, %o0, %o1 ! %o1 gets the difference +7: ! dst aligning loop + add %o1, %o0, %o4 + EX_LD(LOAD(ldub, %o4, %o4), memcpy_retl_o2_plus_o5) ! load one byte + subcc %o5, 1, %o5 + EX_ST(STORE(stb, %o4, %o0), memcpy_retl_o2_plus_o5_plus_1) + bgu,pt %xcc, 7b + add %o0, 1, %o0 ! advance dst + add %o1, %o0, %o1 ! restore %o1 +.Ldst_aligned_on_8: + andcc %o1, 7, %o5 + brnz,pt %o5, .Lsrc_dst_unaligned_on_8 + nop + +.Lsrc_dst_aligned_on_8: + ! check if we are copying MED_MAX or more bytes + set MED_MAX, %o3 + cmp %o2, %o3 ! limit to store buffer size + bgu,pn %xcc, .Llarge_align8_copy + nop + +/* + * Special case for handling when src and dest are both long word aligned + * and total data to move is less than MED_MAX bytes + */ +.Lmedlong: + subcc %o2, 63, %o2 ! adjust length to allow cc test + ble,pn %xcc, .Lmedl63 ! skip big loop if less than 64 bytes + nop +.Lmedl64: + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_63) ! load + subcc %o2, 64, %o2 ! decrement length count + EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_63_64) ! and store + EX_LD(LOAD(ldx, %o1+8, %o3), memcpy_retl_o2_plus_63_56) ! a block of 64 + EX_ST(STORE(stx, %o3, %o0+8), memcpy_retl_o2_plus_63_56) + EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_63_48) + EX_ST(STORE(stx, %o4, %o0+16), memcpy_retl_o2_plus_63_48) + EX_LD(LOAD(ldx, %o1+24, %o3), memcpy_retl_o2_plus_63_40) + EX_ST(STORE(stx, %o3, %o0+24), memcpy_retl_o2_plus_63_40) + EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_63_32)! load and store + EX_ST(STORE(stx, %o4, %o0+32), memcpy_retl_o2_plus_63_32) + EX_LD(LOAD(ldx, %o1+40, %o3), memcpy_retl_o2_plus_63_24)! a block of 64 + add %o1, 64, %o1 ! increase src ptr by 64 + EX_ST(STORE(stx, %o3, %o0+40), memcpy_retl_o2_plus_63_24) + EX_LD(LOAD(ldx, %o1-16, %o4), memcpy_retl_o2_plus_63_16) + add %o0, 64, %o0 ! increase dst ptr by 64 + EX_ST(STORE(stx, %o4, %o0-16), memcpy_retl_o2_plus_63_16) + EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_63_8) + bgu,pt %xcc, .Lmedl64 ! repeat if at least 64 bytes left + EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_63_8) +.Lmedl63: + addcc %o2, 32, %o2 ! adjust remaining count + ble,pt %xcc, .Lmedl31 ! to skip if 31 or fewer bytes left + nop + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_31) ! load + sub %o2, 32, %o2 ! decrement length count + EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_31_32) ! and store + EX_LD(LOAD(ldx, %o1+8, %o3), memcpy_retl_o2_plus_31_24) ! a block of 32 + add %o1, 32, %o1 ! increase src ptr by 32 + EX_ST(STORE(stx, %o3, %o0+8), memcpy_retl_o2_plus_31_24) + EX_LD(LOAD(ldx, %o1-16, %o4), memcpy_retl_o2_plus_31_16) + add %o0, 32, %o0 ! increase dst ptr by 32 + EX_ST(STORE(stx, %o4, %o0-16), memcpy_retl_o2_plus_31_16) + EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_31_8) + EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_31_8) +.Lmedl31: + addcc %o2, 16, %o2 ! adjust remaining count + ble,pt %xcc, .Lmedl15 ! skip if 15 or fewer bytes left + nop ! + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_15) + add %o1, 16, %o1 ! increase src ptr by 16 + EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_15) + sub %o2, 16, %o2 ! decrease count by 16 + EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_15_8) + add %o0, 16, %o0 ! increase dst ptr by 16 + EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_15_8) +.Lmedl15: + addcc %o2, 15, %o2 ! restore count + bz,pt %xcc, .Lsmallx ! exit if finished + cmp %o2, 8 + blt,pt %xcc, .Lmedw7 ! skip if 7 or fewer bytes left + tst %o2 + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2) ! load 8 bytes + add %o1, 8, %o1 ! increase src ptr by 8 + add %o0, 8, %o0 ! increase dst ptr by 8 + subcc %o2, 8, %o2 ! decrease count by 8 + bnz,pn %xcc, .Lmedw7 + EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8) ! and store 8 + retl + mov EX_RETVAL(%g1), %o0 ! restore %o0 + + .align 16 +.Lsrc_dst_unaligned_on_8: + ! DST is 8-byte aligned, src is not +2: + andcc %o1, 0x3, %o5 ! test word alignment + bnz,pt %xcc, .Lunalignsetup ! branch to skip if not word aligned + nop + +/* + * Handle all cases where src and dest are aligned on word + * boundaries. Use unrolled loops for better performance. + * This option wins over standard large data move when + * source and destination is in cache for.Lmedium + * to short data moves. + */ + set MED_WMAX, %o3 + cmp %o2, %o3 ! limit to store buffer size + bge,pt %xcc, .Lunalignrejoin ! otherwise rejoin main loop + nop + + subcc %o2, 31, %o2 ! adjust length to allow cc test + ! for end of loop + ble,pt %xcc, .Lmedw31 ! skip big loop if less than 16 +.Lmedw32: + EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2_plus_31)! move a block of 32 + sllx %o4, 32, %o5 + EX_LD(LOAD(ld, %o1+4, %o4), memcpy_retl_o2_plus_31) + or %o4, %o5, %o5 + EX_ST(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_31) + subcc %o2, 32, %o2 ! decrement length count + EX_LD(LOAD(ld, %o1+8, %o4), memcpy_retl_o2_plus_31_24) + sllx %o4, 32, %o5 + EX_LD(LOAD(ld, %o1+12, %o4), memcpy_retl_o2_plus_31_24) + or %o4, %o5, %o5 + EX_ST(STORE(stx, %o5, %o0+8), memcpy_retl_o2_plus_31_24) + add %o1, 32, %o1 ! increase src ptr by 32 + EX_LD(LOAD(ld, %o1-16, %o4), memcpy_retl_o2_plus_31_16) + sllx %o4, 32, %o5 + EX_LD(LOAD(ld, %o1-12, %o4), memcpy_retl_o2_plus_31_16) + or %o4, %o5, %o5 + EX_ST(STORE(stx, %o5, %o0+16), memcpy_retl_o2_plus_31_16) + add %o0, 32, %o0 ! increase dst ptr by 32 + EX_LD(LOAD(ld, %o1-8, %o4), memcpy_retl_o2_plus_31_8) + sllx %o4, 32, %o5 + EX_LD(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_31_8) + or %o4, %o5, %o5 + bgu,pt %xcc, .Lmedw32 ! repeat if at least 32 bytes left + EX_ST(STORE(stx, %o5, %o0-8), memcpy_retl_o2_plus_31_8) +.Lmedw31: + addcc %o2, 31, %o2 ! restore count + + bz,pt %xcc, .Lsmallx ! exit if finished + nop + cmp %o2, 16 + blt,pt %xcc, .Lmedw15 + nop + EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2)! move a block of 16 bytes + sllx %o4, 32, %o5 + subcc %o2, 16, %o2 ! decrement length count + EX_LD(LOAD(ld, %o1+4, %o4), memcpy_retl_o2_plus_16) + or %o4, %o5, %o5 + EX_ST(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_16) + add %o1, 16, %o1 ! increase src ptr by 16 + EX_LD(LOAD(ld, %o1-8, %o4), memcpy_retl_o2_plus_8) + add %o0, 16, %o0 ! increase dst ptr by 16 + sllx %o4, 32, %o5 + EX_LD(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_8) + or %o4, %o5, %o5 + EX_ST(STORE(stx, %o5, %o0-8), memcpy_retl_o2_plus_8) +.Lmedw15: + bz,pt %xcc, .Lsmallx ! exit if finished + cmp %o2, 8 + blt,pn %xcc, .Lmedw7 ! skip if 7 or fewer bytes left + tst %o2 + EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2) ! load 4 bytes + subcc %o2, 8, %o2 ! decrease count by 8 + EX_ST(STORE(stw, %o4, %o0), memcpy_retl_o2_plus_8)! and store 4 bytes + add %o1, 8, %o1 ! increase src ptr by 8 + EX_LD(LOAD(ld, %o1-4, %o3), memcpy_retl_o2_plus_4) ! load 4 bytes + add %o0, 8, %o0 ! increase dst ptr by 8 + EX_ST(STORE(stw, %o3, %o0-4), memcpy_retl_o2_plus_4)! and store 4 bytes + bz,pt %xcc, .Lsmallx ! exit if finished +.Lmedw7: ! count is ge 1, less than 8 + cmp %o2, 4 ! check for 4 bytes left + blt,pn %xcc, .Lsmallleft3 ! skip if 3 or fewer bytes left + nop ! + EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2) ! load 4 bytes + add %o1, 4, %o1 ! increase src ptr by 4 + add %o0, 4, %o0 ! increase dst ptr by 4 + subcc %o2, 4, %o2 ! decrease count by 4 + bnz .Lsmallleft3 + EX_ST(STORE(stw, %o4, %o0-4), memcpy_retl_o2_plus_4)! and store 4 bytes + retl + mov EX_RETVAL(%g1), %o0 + + .align 16 +.Llarge_align8_copy: ! Src and dst share 8 byte alignment + ! align dst to 64 byte boundary + andcc %o0, 0x3f, %o3 ! %o3 == 0 means dst is 64 byte aligned + brz,pn %o3, .Laligned_to_64 + andcc %o0, 8, %o3 ! odd long words to move? + brz,pt %o3, .Laligned_to_16 + nop + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2) + sub %o2, 8, %o2 + add %o1, 8, %o1 ! increment src ptr + add %o0, 8, %o0 ! increment dst ptr + EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8) +.Laligned_to_16: + andcc %o0, 16, %o3 ! pair of long words to move? + brz,pt %o3, .Laligned_to_32 + nop + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2) + sub %o2, 16, %o2 + EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_16) + add %o1, 16, %o1 ! increment src ptr + EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_8) + add %o0, 16, %o0 ! increment dst ptr + EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8) +.Laligned_to_32: + andcc %o0, 32, %o3 ! four long words to move? + brz,pt %o3, .Laligned_to_64 + nop + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2) + sub %o2, 32, %o2 + EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_32) + EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_24) + EX_ST(STORE(stx, %o4, %o0+8), memcpy_retl_o2_plus_24) + EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_16) + EX_ST(STORE(stx, %o4, %o0+16), memcpy_retl_o2_plus_16) + add %o1, 32, %o1 ! increment src ptr + EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_8) + add %o0, 32, %o0 ! increment dst ptr + EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8) +.Laligned_to_64: +! +! Using block init store (BIS) instructions to avoid fetching cache +! lines from memory. Use ST_CHUNK stores to first element of each cache +! line (similar to prefetching) to avoid overfilling STQ or miss buffers. +! Gives existing cache lines time to be moved out of L1/L2/L3 cache. +! Initial stores using MRU version of BIS to keep cache line in +! cache until we are ready to store final element of cache line. +! Then store last element using the LRU version of BIS. +! + andn %o2, 0x3f, %o5 ! %o5 is multiple of block size + and %o2, 0x3f, %o2 ! residue bytes in %o2 +! +! We use STORE_MRU_ASI for the first seven stores to each cache line +! followed by STORE_ASI (mark as LRU) for the last store. That +! mixed approach reduces the probability that the cache line is removed +! before we finish setting it, while minimizing the effects on +! other cached values during a large memcpy +! +! ST_CHUNK batches up initial BIS operations for several cache lines +! to allow multiple requests to not be blocked by overflowing the +! the store miss buffer. Then the matching stores for all those +! BIS operations are executed. +! + + sub %o0, 8, %o0 ! adjust %o0 for ASI alignment +.Lalign_loop: + cmp %o5, ST_CHUNK*64 + blu,pt %xcc, .Lalign_loop_fin + mov ST_CHUNK,%o3 +.Lalign_loop_start: + prefetch [%o1 + (ALIGN_PRE * BLOCK_SIZE)], 21 + subcc %o3, 1, %o3 + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_o5) + add %o1, 64, %o1 + add %o0, 8, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + bgu %xcc,.Lalign_loop_start + add %o0, 56, %o0 + + mov ST_CHUNK,%o3 + sllx %o3, 6, %o4 ! ST_CHUNK*64 + sub %o1, %o4, %o1 ! reset %o1 + sub %o0, %o4, %o0 ! reset %o0 + +.Lalign_loop_rest: + EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_o5) + add %o0, 16, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_o5) + add %o0, 8, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + subcc %o3, 1, %o3 + EX_LD(LOAD(ldx, %o1+24, %o4), memcpy_retl_o2_plus_o5) + add %o0, 8, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_o5) + add %o0, 8, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1+40, %o4), memcpy_retl_o2_plus_o5) + add %o0, 8, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1+48, %o4), memcpy_retl_o2_plus_o5) + add %o1, 64, %o1 + add %o0, 8, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + add %o0, 8, %o0 + EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_o5) + sub %o5, 64, %o5 + bgu %xcc,.Lalign_loop_rest + ! mark cache line as LRU + EX_ST(STORE_INIT(%o4, %o0), memcpy_retl_o2_plus_o5_plus_64) + + cmp %o5, ST_CHUNK*64 + bgu,pt %xcc, .Lalign_loop_start + mov ST_CHUNK,%o3 + + cmp %o5, 0 + beq .Lalign_done + nop +.Lalign_loop_fin: + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_o5) + EX_ST(STORE(stx, %o4, %o0+8), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_o5) + EX_ST(STORE(stx, %o4, %o0+8+8), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_o5) + EX_ST(STORE(stx, %o4, %o0+8+16), memcpy_retl_o2_plus_o5) + subcc %o5, 64, %o5 + EX_LD(LOAD(ldx, %o1+24, %o4), memcpy_retl_o2_plus_o5_64) + EX_ST(STORE(stx, %o4, %o0+8+24), memcpy_retl_o2_plus_o5_64) + EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_o5_64) + EX_ST(STORE(stx, %o4, %o0+8+32), memcpy_retl_o2_plus_o5_64) + EX_LD(LOAD(ldx, %o1+40, %o4), memcpy_retl_o2_plus_o5_64) + EX_ST(STORE(stx, %o4, %o0+8+40), memcpy_retl_o2_plus_o5_64) + EX_LD(LOAD(ldx, %o1+48, %o4), memcpy_retl_o2_plus_o5_64) + add %o1, 64, %o1 + EX_ST(STORE(stx, %o4, %o0+8+48), memcpy_retl_o2_plus_o5_64) + add %o0, 64, %o0 + EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_o5_64) + bgu %xcc,.Lalign_loop_fin + EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_o5_64) + +.Lalign_done: + add %o0, 8, %o0 ! restore %o0 from ASI alignment + membar #StoreStore + sub %o2, 63, %o2 ! adjust length to allow cc test + ba .Lmedl63 ! in .Lmedl63 + nop + + .align 16 + ! Dst is on 8 byte boundary; src is not; remaining count > SMALL_MAX +.Lunalignsetup: +.Lunalignrejoin: + mov %g1, %o3 ! save %g1 as VISEntryHalf clobbers it +#ifdef NON_USER_COPY + VISEntryHalfFast(.Lmedium_vis_entry_fail_cp) +#else + VISEntryHalf +#endif + mov %o3, %g1 ! restore %g1 + + set MED_UMAX, %o3 + cmp %o2, %o3 ! check for.Lmedium unaligned limit + bge,pt %xcc,.Lunalign_large + prefetch [%o1 + (4 * BLOCK_SIZE)], 20 + andn %o2, 0x3f, %o5 ! %o5 is multiple of block size + and %o2, 0x3f, %o2 ! residue bytes in %o2 + cmp %o2, 8 ! Insure we do not load beyond + bgt .Lunalign_adjust ! end of source buffer + andn %o1, 0x7, %o4 ! %o4 has long word aligned src address + add %o2, 64, %o2 ! adjust to leave loop + sub %o5, 64, %o5 ! early if necessary +.Lunalign_adjust: + alignaddr %o1, %g0, %g0 ! generate %gsr + add %o1, %o5, %o1 ! advance %o1 to after blocks + EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5) +.Lunalign_loop: + EX_LD_FP(LOAD(ldd, %o4+8, %f2), memcpy_retl_o2_plus_o5) + faligndata %f0, %f2, %f16 + EX_LD_FP(LOAD(ldd, %o4+16, %f4), memcpy_retl_o2_plus_o5) + subcc %o5, BLOCK_SIZE, %o5 + EX_ST_FP(STORE(std, %f16, %o0), memcpy_retl_o2_plus_o5_plus_64) + faligndata %f2, %f4, %f18 + EX_LD_FP(LOAD(ldd, %o4+24, %f6), memcpy_retl_o2_plus_o5_plus_56) + EX_ST_FP(STORE(std, %f18, %o0+8), memcpy_retl_o2_plus_o5_plus_56) + faligndata %f4, %f6, %f20 + EX_LD_FP(LOAD(ldd, %o4+32, %f8), memcpy_retl_o2_plus_o5_plus_48) + EX_ST_FP(STORE(std, %f20, %o0+16), memcpy_retl_o2_plus_o5_plus_48) + faligndata %f6, %f8, %f22 + EX_LD_FP(LOAD(ldd, %o4+40, %f10), memcpy_retl_o2_plus_o5_plus_40) + EX_ST_FP(STORE(std, %f22, %o0+24), memcpy_retl_o2_plus_o5_plus_40) + faligndata %f8, %f10, %f24 + EX_LD_FP(LOAD(ldd, %o4+48, %f12), memcpy_retl_o2_plus_o5_plus_32) + EX_ST_FP(STORE(std, %f24, %o0+32), memcpy_retl_o2_plus_o5_plus_32) + faligndata %f10, %f12, %f26 + EX_LD_FP(LOAD(ldd, %o4+56, %f14), memcpy_retl_o2_plus_o5_plus_24) + add %o4, BLOCK_SIZE, %o4 + EX_ST_FP(STORE(std, %f26, %o0+40), memcpy_retl_o2_plus_o5_plus_24) + faligndata %f12, %f14, %f28 + EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5_plus_16) + EX_ST_FP(STORE(std, %f28, %o0+48), memcpy_retl_o2_plus_o5_plus_16) + faligndata %f14, %f0, %f30 + EX_ST_FP(STORE(std, %f30, %o0+56), memcpy_retl_o2_plus_o5_plus_8) + add %o0, BLOCK_SIZE, %o0 + bgu,pt %xcc, .Lunalign_loop + prefetch [%o4 + (5 * BLOCK_SIZE)], 20 + ba .Lunalign_done + nop + +.Lunalign_large: + andcc %o0, 0x3f, %o3 ! is dst 64-byte block aligned? + bz %xcc, .Lunalignsrc + sub %o3, 64, %o3 ! %o3 will be multiple of 8 + neg %o3 ! bytes until dest is 64 byte aligned + sub %o2, %o3, %o2 ! update cnt with bytes to be moved + ! Move bytes according to source alignment + andcc %o1, 0x1, %o5 + bnz %xcc, .Lunalignbyte ! check for byte alignment + nop + andcc %o1, 2, %o5 ! check for half word alignment + bnz %xcc, .Lunalignhalf + nop + ! Src is word aligned +.Lunalignword: + EX_LD_FP(LOAD(ld, %o1, %o4), memcpy_retl_o2_plus_o3) ! load 4 bytes + add %o1, 8, %o1 ! increase src ptr by 8 + EX_ST_FP(STORE(stw, %o4, %o0), memcpy_retl_o2_plus_o3) ! and store 4 + subcc %o3, 8, %o3 ! decrease count by 8 + EX_LD_FP(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_o3_plus_4)! load 4 + add %o0, 8, %o0 ! increase dst ptr by 8 + bnz %xcc, .Lunalignword + EX_ST_FP(STORE(stw, %o4, %o0-4), memcpy_retl_o2_plus_o3_plus_4) + ba .Lunalignsrc + nop + + ! Src is half-word aligned +.Lunalignhalf: + EX_LD_FP(LOAD(lduh, %o1, %o4), memcpy_retl_o2_plus_o3) ! load 2 bytes + sllx %o4, 32, %o5 ! shift left + EX_LD_FP(LOAD(lduw, %o1+2, %o4), memcpy_retl_o2_plus_o3) + or %o4, %o5, %o5 + sllx %o5, 16, %o5 + EX_LD_FP(LOAD(lduh, %o1+6, %o4), memcpy_retl_o2_plus_o3) + or %o4, %o5, %o5 + EX_ST_FP(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_o3) + add %o1, 8, %o1 + subcc %o3, 8, %o3 + bnz %xcc, .Lunalignhalf + add %o0, 8, %o0 + ba .Lunalignsrc + nop + + ! Src is Byte aligned +.Lunalignbyte: + sub %o0, %o1, %o0 ! share pointer advance +.Lunalignbyte_loop: + EX_LD_FP(LOAD(ldub, %o1, %o4), memcpy_retl_o2_plus_o3) + sllx %o4, 56, %o5 + EX_LD_FP(LOAD(lduh, %o1+1, %o4), memcpy_retl_o2_plus_o3) + sllx %o4, 40, %o4 + or %o4, %o5, %o5 + EX_LD_FP(LOAD(lduh, %o1+3, %o4), memcpy_retl_o2_plus_o3) + sllx %o4, 24, %o4 + or %o4, %o5, %o5 + EX_LD_FP(LOAD(lduh, %o1+5, %o4), memcpy_retl_o2_plus_o3) + sllx %o4, 8, %o4 + or %o4, %o5, %o5 + EX_LD_FP(LOAD(ldub, %o1+7, %o4), memcpy_retl_o2_plus_o3) + or %o4, %o5, %o5 + add %o0, %o1, %o0 + EX_ST_FP(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_o3) + sub %o0, %o1, %o0 + subcc %o3, 8, %o3 + bnz %xcc, .Lunalignbyte_loop + add %o1, 8, %o1 + add %o0,%o1, %o0 ! restore pointer + + ! Destination is now block (64 byte aligned) +.Lunalignsrc: + andn %o2, 0x3f, %o5 ! %o5 is multiple of block size + and %o2, 0x3f, %o2 ! residue bytes in %o2 + add %o2, 64, %o2 ! Insure we do not load beyond + sub %o5, 64, %o5 ! end of source buffer + + andn %o1, 0x7, %o4 ! %o4 has long word aligned src address + alignaddr %o1, %g0, %g0 ! generate %gsr + add %o1, %o5, %o1 ! advance %o1 to after blocks + + EX_LD_FP(LOAD(ldd, %o4, %f14), memcpy_retl_o2_plus_o5) + add %o4, 8, %o4 +.Lunalign_sloop: + EX_LD_FP(LOAD(ldd, %o4, %f16), memcpy_retl_o2_plus_o5) + faligndata %f14, %f16, %f0 + EX_LD_FP(LOAD(ldd, %o4+8, %f18), memcpy_retl_o2_plus_o5) + faligndata %f16, %f18, %f2 + EX_LD_FP(LOAD(ldd, %o4+16, %f20), memcpy_retl_o2_plus_o5) + faligndata %f18, %f20, %f4 + EX_ST_FP(STORE(std, %f0, %o0), memcpy_retl_o2_plus_o5) + subcc %o5, 64, %o5 + EX_LD_FP(LOAD(ldd, %o4+24, %f22), memcpy_retl_o2_plus_o5_plus_56) + faligndata %f20, %f22, %f6 + EX_ST_FP(STORE(std, %f2, %o0+8), memcpy_retl_o2_plus_o5_plus_56) + EX_LD_FP(LOAD(ldd, %o4+32, %f24), memcpy_retl_o2_plus_o5_plus_48) + faligndata %f22, %f24, %f8 + EX_ST_FP(STORE(std, %f4, %o0+16), memcpy_retl_o2_plus_o5_plus_48) + EX_LD_FP(LOAD(ldd, %o4+40, %f26), memcpy_retl_o2_plus_o5_plus_40) + faligndata %f24, %f26, %f10 + EX_ST_FP(STORE(std, %f6, %o0+24), memcpy_retl_o2_plus_o5_plus_40) + EX_LD_FP(LOAD(ldd, %o4+48, %f28), memcpy_retl_o2_plus_o5_plus_32) + faligndata %f26, %f28, %f12 + EX_ST_FP(STORE(std, %f8, %o0+32), memcpy_retl_o2_plus_o5_plus_32) + add %o4, 64, %o4 + EX_LD_FP(LOAD(ldd, %o4-8, %f30), memcpy_retl_o2_plus_o5_plus_24) + faligndata %f28, %f30, %f14 + EX_ST_FP(STORE(std, %f10, %o0+40), memcpy_retl_o2_plus_o5_plus_24) + EX_ST_FP(STORE(std, %f12, %o0+48), memcpy_retl_o2_plus_o5_plus_16) + add %o0, 64, %o0 + EX_ST_FP(STORE(std, %f14, %o0-8), memcpy_retl_o2_plus_o5_plus_8) + fsrc2 %f30, %f14 + bgu,pt %xcc, .Lunalign_sloop + prefetch [%o4 + (8 * BLOCK_SIZE)], 20 + +.Lunalign_done: + ! Handle trailing bytes, 64 to 127 + ! Dest long word aligned, Src not long word aligned + cmp %o2, 15 + bleu %xcc, .Lunalign_short + + andn %o2, 0x7, %o5 ! %o5 is multiple of 8 + and %o2, 0x7, %o2 ! residue bytes in %o2 + add %o2, 8, %o2 + sub %o5, 8, %o5 ! insure we do not load past end of src + andn %o1, 0x7, %o4 ! %o4 has long word aligned src address + add %o1, %o5, %o1 ! advance %o1 to after multiple of 8 + EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5)! fetch partialword +.Lunalign_by8: + EX_LD_FP(LOAD(ldd, %o4+8, %f2), memcpy_retl_o2_plus_o5) + add %o4, 8, %o4 + faligndata %f0, %f2, %f16 + subcc %o5, 8, %o5 + EX_ST_FP(STORE(std, %f16, %o0), memcpy_retl_o2_plus_o5_plus_8) + fsrc2 %f2, %f0 + bgu,pt %xcc, .Lunalign_by8 + add %o0, 8, %o0 + +.Lunalign_short: +#ifdef NON_USER_COPY + VISExitHalfFast +#else + VISExitHalf +#endif + ba .Lsmallrest + nop + +/* + * This is a special case of nested memcpy. This can happen when kernel + * calls unaligned memcpy back to back without saving FP registers. We need + * traps(context switch) to save/restore FP registers. If the kernel calls + * memcpy without this trap sequence we will hit FP corruption. Let's use + * the normal integer load/store method in this case. + */ + +#ifdef NON_USER_COPY +.Lmedium_vis_entry_fail_cp: + or %o0, %o1, %g2 +#endif +.Lmedium_cp: + LOAD(prefetch, %o1 + 0x40, #n_reads_strong) + andcc %g2, 0x7, %g0 + bne,pn %xcc, .Lmedium_unaligned_cp + nop + +.Lmedium_noprefetch_cp: + andncc %o2, 0x20 - 1, %o5 + be,pn %xcc, 2f + sub %o2, %o5, %o2 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x08, %g2), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x10, %g7), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x18, %o4), memcpy_retl_o2_plus_o5) + add %o1, 0x20, %o1 + subcc %o5, 0x20, %o5 + EX_ST(STORE(stx, %o3, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_32) + EX_ST(STORE(stx, %g2, %o0 + 0x08), memcpy_retl_o2_plus_o5_plus_24) + EX_ST(STORE(stx, %g7, %o0 + 0x10), memcpy_retl_o2_plus_o5_plus_16) + EX_ST(STORE(stx, %o4, %o0 + 0x18), memcpy_retl_o2_plus_o5_plus_8) + bne,pt %xcc, 1b + add %o0, 0x20, %o0 +2: andcc %o2, 0x18, %o5 + be,pt %xcc, 3f + sub %o2, %o5, %o2 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5) + add %o1, 0x08, %o1 + add %o0, 0x08, %o0 + subcc %o5, 0x08, %o5 + bne,pt %xcc, 1b + EX_ST(STORE(stx, %o3, %o0 - 0x08), memcpy_retl_o2_plus_o5_plus_8) +3: brz,pt %o2, .Lexit_cp + cmp %o2, 0x04 + bl,pn %xcc, .Ltiny_cp + nop + EX_LD(LOAD(lduw, %o1 + 0x00, %o3), memcpy_retl_o2) + add %o1, 0x04, %o1 + add %o0, 0x04, %o0 + subcc %o2, 0x04, %o2 + bne,pn %xcc, .Ltiny_cp + EX_ST(STORE(stw, %o3, %o0 - 0x04), memcpy_retl_o2_plus_4) + ba,a,pt %xcc, .Lexit_cp + +.Lmedium_unaligned_cp: + /* First get dest 8 byte aligned. */ + sub %g0, %o0, %o3 + and %o3, 0x7, %o3 + brz,pt %o3, 2f + sub %o2, %o3, %o2 + +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), memcpy_retl_o2_plus_o3) + add %o1, 1, %o1 + subcc %o3, 1, %o3 + add %o0, 1, %o0 + bne,pt %xcc, 1b + EX_ST(STORE(stb, %g2, %o0 - 0x01), memcpy_retl_o2_plus_o3_plus_1) +2: + and %o1, 0x7, %o3 + brz,pn %o3, .Lmedium_noprefetch_cp + sll %o3, 3, %o3 + mov 64, %g2 + sub %g2, %o3, %g2 + andn %o1, 0x7, %o1 + EX_LD(LOAD(ldx, %o1 + 0x00, %o4), memcpy_retl_o2) + sllx %o4, %o3, %o4 + andn %o2, 0x08 - 1, %o5 + sub %o2, %o5, %o2 + +1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), memcpy_retl_o2_plus_o5) + add %o1, 0x08, %o1 + subcc %o5, 0x08, %o5 + srlx %g3, %g2, %g7 + or %g7, %o4, %g7 + EX_ST(STORE(stx, %g7, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_8) + add %o0, 0x08, %o0 + bne,pt %xcc, 1b + sllx %g3, %o3, %o4 + srl %o3, 3, %o3 + add %o1, %o3, %o1 + brz,pn %o2, .Lexit_cp + nop + ba,pt %xcc, .Lsmall_unaligned_cp + +.Ltiny_cp: + EX_LD(LOAD(ldub, %o1 + 0x00, %o3), memcpy_retl_o2) + subcc %o2, 1, %o2 + be,pn %xcc, .Lexit_cp + EX_ST(STORE(stb, %o3, %o0 + 0x00), memcpy_retl_o2_plus_1) + EX_LD(LOAD(ldub, %o1 + 0x01, %o3), memcpy_retl_o2) + subcc %o2, 1, %o2 + be,pn %xcc, .Lexit_cp + EX_ST(STORE(stb, %o3, %o0 + 0x01), memcpy_retl_o2_plus_1) + EX_LD(LOAD(ldub, %o1 + 0x02, %o3), memcpy_retl_o2) + ba,pt %xcc, .Lexit_cp + EX_ST(STORE(stb, %o3, %o0 + 0x02), memcpy_retl_o2) + +.Lsmall_cp: + andcc %g2, 0x3, %g0 + bne,pn %xcc, .Lsmall_unaligned_cp + andn %o2, 0x4 - 1, %o5 + sub %o2, %o5, %o2 +1: + EX_LD(LOAD(lduw, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5) + add %o1, 0x04, %o1 + subcc %o5, 0x04, %o5 + add %o0, 0x04, %o0 + bne,pt %xcc, 1b + EX_ST(STORE(stw, %o3, %o0 - 0x04), memcpy_retl_o2_plus_o5_plus_4) + brz,pt %o2, .Lexit_cp + nop + ba,a,pt %xcc, .Ltiny_cp + +.Lsmall_unaligned_cp: +1: EX_LD(LOAD(ldub, %o1 + 0x00, %o3), memcpy_retl_o2) + add %o1, 1, %o1 + add %o0, 1, %o0 + subcc %o2, 1, %o2 + bne,pt %xcc, 1b + EX_ST(STORE(stb, %o3, %o0 - 0x01), memcpy_retl_o2_plus_1) + ba,a,pt %xcc, .Lexit_cp + +.Lsmallrest: + tst %o2 + bz,pt %xcc, .Lsmallx + cmp %o2, 4 + blt,pn %xcc, .Lsmallleft3 + nop + sub %o2, 3, %o2 +.Lsmallnotalign4: + EX_LD(LOAD(ldub, %o1, %o3), memcpy_retl_o2_plus_3)! read byte + subcc %o2, 4, %o2 ! reduce count by 4 + EX_ST(STORE(stb, %o3, %o0), memcpy_retl_o2_plus_7)! write byte & repeat + EX_LD(LOAD(ldub, %o1+1, %o3), memcpy_retl_o2_plus_6)! for total of 4 + add %o1, 4, %o1 ! advance SRC by 4 + EX_ST(STORE(stb, %o3, %o0+1), memcpy_retl_o2_plus_6) + EX_LD(LOAD(ldub, %o1-2, %o3), memcpy_retl_o2_plus_5) + add %o0, 4, %o0 ! advance DST by 4 + EX_ST(STORE(stb, %o3, %o0-2), memcpy_retl_o2_plus_5) + EX_LD(LOAD(ldub, %o1-1, %o3), memcpy_retl_o2_plus_4) + bgu,pt %xcc, .Lsmallnotalign4 ! loop til 3 or fewer bytes remain + EX_ST(STORE(stb, %o3, %o0-1), memcpy_retl_o2_plus_4) + addcc %o2, 3, %o2 ! restore count + bz,pt %xcc, .Lsmallx +.Lsmallleft3: ! 1, 2, or 3 bytes remain + subcc %o2, 1, %o2 + EX_LD(LOAD(ldub, %o1, %o3), memcpy_retl_o2_plus_1) ! load one byte + bz,pt %xcc, .Lsmallx + EX_ST(STORE(stb, %o3, %o0), memcpy_retl_o2_plus_1) ! store one byte + EX_LD(LOAD(ldub, %o1+1, %o3), memcpy_retl_o2) ! load second byte + subcc %o2, 1, %o2 + bz,pt %xcc, .Lsmallx + EX_ST(STORE(stb, %o3, %o0+1), memcpy_retl_o2_plus_1)! store second byte + EX_LD(LOAD(ldub, %o1+2, %o3), memcpy_retl_o2) ! load third byte + EX_ST(STORE(stb, %o3, %o0+2), memcpy_retl_o2) ! store third byte +.Lsmallx: + retl + mov EX_RETVAL(%g1), %o0 +.Lsmallfin: + tst %o2 + bnz,pn %xcc, .Lsmallleft3 + nop + retl + mov EX_RETVAL(%g1), %o0 ! restore %o0 +.Lexit_cp: + retl + mov EX_RETVAL(%g1), %o0 + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/M7memset.S b/arch/sparc/lib/M7memset.S new file mode 100644 index 000000000000..62ea91b3a6b8 --- /dev/null +++ b/arch/sparc/lib/M7memset.S @@ -0,0 +1,352 @@ +/* + * M7memset.S: SPARC M7 optimized memset. + * + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + */ + +/* + * M7memset.S: M7 optimized memset. + * + * char *memset(sp, c, n) + * + * Set an array of n chars starting at sp to the character c. + * Return sp. + * + * Fast assembler language version of the following C-program for memset + * which represents the `standard' for the C-library. + * + * void * + * memset(void *sp1, int c, size_t n) + * { + * if (n != 0) { + * char *sp = sp1; + * do { + * *sp++ = (char)c; + * } while (--n != 0); + * } + * return (sp1); + * } + * + * The algorithm is as follows : + * + * For small 6 or fewer bytes stores, bytes will be stored. + * + * For less than 32 bytes stores, align the address on 4 byte boundary. + * Then store as many 4-byte chunks, followed by trailing bytes. + * + * For sizes greater than 32 bytes, align the address on 8 byte boundary. + * if (count >= 64) { + * store 8-bytes chunks to align the address on 64 byte boundary + * if (value to be set is zero && count >= MIN_ZERO) { + * Using BIS stores, set the first long word of each + * 64-byte cache line to zero which will also clear the + * other seven long words of the cache line. + * } + * else if (count >= MIN_LOOP) { + * Using BIS stores, set the first long word of each of + * ST_CHUNK cache lines (64 bytes each) before the main + * loop is entered. + * In the main loop, continue pre-setting the first long + * word of each cache line ST_CHUNK lines in advance while + * setting the other seven long words (56 bytes) of each + * cache line until fewer than ST_CHUNK*64 bytes remain. + * Then set the remaining seven long words of each cache + * line that has already had its first long word set. + * } + * store remaining data in 64-byte chunks until less than + * 64 bytes remain. + * } + * Store as many 8-byte chunks, followed by trailing bytes. + * + * BIS = Block Init Store + * Doing the advance store of the first element of the cache line + * initiates the displacement of a cache line while only using a single + * instruction in the pipeline. That avoids various pipeline delays, + * such as filling the miss buffer. The performance effect is + * similar to prefetching for normal stores. + * The special case for zero fills runs faster and uses fewer instruction + * cycles than the normal memset loop. + * + * We only use BIS for memset of greater than MIN_LOOP bytes because a sequence + * BIS stores must be followed by a membar #StoreStore. The benefit of + * the BIS store must be balanced against the cost of the membar operation. + */ + +/* + * ASI_STBI_P marks the cache line as "least recently used" + * which means if many threads are active, it has a high chance + * of being pushed out of the cache between the first initializing + * store and the final stores. + * Thus, we use ASI_STBIMRU_P which marks the cache line as + * "most recently used" for all but the last store to the cache line. + */ + +#include <asm/asi.h> +#include <asm/page.h> + +#define ASI_STBI_P ASI_BLK_INIT_QUAD_LDD_P +#define ASI_STBIMRU_P ASI_ST_BLKINIT_MRU_P + + +#define ST_CHUNK 24 /* multiple of 4 due to loop unrolling */ +#define MIN_LOOP 16320 +#define MIN_ZERO 512 + + .section ".text" + .align 32 + +/* + * Define clear_page(dest) as memset(dest, 0, PAGE_SIZE) + * (can create a more optimized version later.) + */ + .globl M7clear_page + .globl M7clear_user_page +M7clear_page: /* clear_page(dest) */ +M7clear_user_page: + set PAGE_SIZE, %o1 + /* fall through into bzero code */ + + .size M7clear_page,.-M7clear_page + .size M7clear_user_page,.-M7clear_user_page + +/* + * Define bzero(dest, n) as memset(dest, 0, n) + * (can create a more optimized version later.) + */ + .globl M7bzero +M7bzero: /* bzero(dest, size) */ + mov %o1, %o2 + mov 0, %o1 + /* fall through into memset code */ + + .size M7bzero,.-M7bzero + + .global M7memset + .type M7memset, #function + .register %g3, #scratch +M7memset: + mov %o0, %o5 ! copy sp1 before using it + cmp %o2, 7 ! if small counts, just write bytes + bleu,pn %xcc, .wrchar + and %o1, 0xff, %o1 ! o1 is (char)c + + sll %o1, 8, %o3 + or %o1, %o3, %o1 ! now o1 has 2 bytes of c + sll %o1, 16, %o3 + cmp %o2, 32 + blu,pn %xcc, .wdalign + or %o1, %o3, %o1 ! now o1 has 4 bytes of c + + sllx %o1, 32, %o3 + or %o1, %o3, %o1 ! now o1 has 8 bytes of c + +.dbalign: + andcc %o5, 7, %o3 ! is sp1 aligned on a 8 byte bound? + bz,pt %xcc, .blkalign ! already long word aligned + sub %o3, 8, %o3 ! -(bytes till long word aligned) + + add %o2, %o3, %o2 ! update o2 with new count + ! Set -(%o3) bytes till sp1 long word aligned +1: stb %o1, [%o5] ! there is at least 1 byte to set + inccc %o3 ! byte clearing loop + bl,pt %xcc, 1b + inc %o5 + + ! Now sp1 is long word aligned (sp1 is found in %o5) +.blkalign: + cmp %o2, 64 ! check if there are 64 bytes to set + blu,pn %xcc, .wrshort + mov %o2, %o3 + + andcc %o5, 63, %o3 ! is sp1 block aligned? + bz,pt %xcc, .blkwr ! now block aligned + sub %o3, 64, %o3 ! o3 is -(bytes till block aligned) + add %o2, %o3, %o2 ! o2 is the remainder + + ! Store -(%o3) bytes till dst is block (64 byte) aligned. + ! Use long word stores. + ! Recall that dst is already long word aligned +1: + addcc %o3, 8, %o3 + stx %o1, [%o5] + bl,pt %xcc, 1b + add %o5, 8, %o5 + + ! Now sp1 is block aligned +.blkwr: + andn %o2, 63, %o4 ! calculate size of blocks in bytes + brz,pn %o1, .wrzero ! special case if c == 0 + and %o2, 63, %o3 ! %o3 = bytes left after blk stores. + + set MIN_LOOP, %g1 + cmp %o4, %g1 ! check there are enough bytes to set + blu,pn %xcc, .short_set ! to justify cost of membar + ! must be > pre-cleared lines + nop + + ! initial cache-clearing stores + ! get store pipeline moving + rd %asi, %g3 ! save %asi to be restored later + wr %g0, ASI_STBIMRU_P, %asi + + ! Primary memset loop for large memsets +.wr_loop: + sub %o5, 8, %o5 ! adjust %o5 for ASI store alignment + mov ST_CHUNK, %g1 +.wr_loop_start: + stxa %o1, [%o5+8]%asi + subcc %g1, 4, %g1 + stxa %o1, [%o5+8+64]%asi + add %o5, 256, %o5 + stxa %o1, [%o5+8-128]%asi + bgu %xcc, .wr_loop_start + stxa %o1, [%o5+8-64]%asi + + sub %o5, ST_CHUNK*64, %o5 ! reset %o5 + mov ST_CHUNK, %g1 + +.wr_loop_rest: + stxa %o1, [%o5+8+8]%asi + sub %o4, 64, %o4 + stxa %o1, [%o5+16+8]%asi + subcc %g1, 1, %g1 + stxa %o1, [%o5+24+8]%asi + stxa %o1, [%o5+32+8]%asi + stxa %o1, [%o5+40+8]%asi + add %o5, 64, %o5 + stxa %o1, [%o5-8]%asi + bgu %xcc, .wr_loop_rest + stxa %o1, [%o5]ASI_STBI_P + + ! If more than ST_CHUNK*64 bytes remain to set, continue + ! setting the first long word of each cache line in advance + ! to keep the store pipeline moving. + + cmp %o4, ST_CHUNK*64 + bge,pt %xcc, .wr_loop_start + mov ST_CHUNK, %g1 + + brz,a,pn %o4, .asi_done + add %o5, 8, %o5 ! restore %o5 offset + +.wr_loop_small: + stxa %o1, [%o5+8]%asi + stxa %o1, [%o5+8+8]%asi + stxa %o1, [%o5+16+8]%asi + stxa %o1, [%o5+24+8]%asi + stxa %o1, [%o5+32+8]%asi + subcc %o4, 64, %o4 + stxa %o1, [%o5+40+8]%asi + add %o5, 64, %o5 + stxa %o1, [%o5-8]%asi + bgu,pt %xcc, .wr_loop_small + stxa %o1, [%o5]ASI_STBI_P + + ba .asi_done + add %o5, 8, %o5 ! restore %o5 offset + + ! Special case loop for zero fill memsets + ! For each 64 byte cache line, single STBI to first element + ! clears line +.wrzero: + cmp %o4, MIN_ZERO ! check if enough bytes to set + ! to pay %asi + membar cost + blu %xcc, .short_set + nop + sub %o4, 256, %o4 + +.wrzero_loop: + mov 64, %g3 + stxa %o1, [%o5]ASI_STBI_P + subcc %o4, 256, %o4 + stxa %o1, [%o5+%g3]ASI_STBI_P + add %o5, 256, %o5 + sub %g3, 192, %g3 + stxa %o1, [%o5+%g3]ASI_STBI_P + add %g3, 64, %g3 + bge,pt %xcc, .wrzero_loop + stxa %o1, [%o5+%g3]ASI_STBI_P + add %o4, 256, %o4 + + brz,pn %o4, .bsi_done + nop + +.wrzero_small: + stxa %o1, [%o5]ASI_STBI_P + subcc %o4, 64, %o4 + bgu,pt %xcc, .wrzero_small + add %o5, 64, %o5 + ba,a .bsi_done + +.asi_done: + wr %g3, 0x0, %asi ! restored saved %asi +.bsi_done: + membar #StoreStore ! required by use of Block Store Init + +.short_set: + cmp %o4, 64 ! check if 64 bytes to set + blu %xcc, 5f + nop +4: ! set final blocks of 64 bytes + stx %o1, [%o5] + stx %o1, [%o5+8] + stx %o1, [%o5+16] + stx %o1, [%o5+24] + subcc %o4, 64, %o4 + stx %o1, [%o5+32] + stx %o1, [%o5+40] + add %o5, 64, %o5 + stx %o1, [%o5-16] + bgu,pt %xcc, 4b + stx %o1, [%o5-8] + +5: + ! Set the remaining long words +.wrshort: + subcc %o3, 8, %o3 ! Can we store any long words? + blu,pn %xcc, .wrchars + and %o2, 7, %o2 ! calc bytes left after long words +6: + subcc %o3, 8, %o3 + stx %o1, [%o5] ! store the long words + bgeu,pt %xcc, 6b + add %o5, 8, %o5 + +.wrchars: ! check for extra chars + brnz %o2, .wrfin + nop + retl + nop + +.wdalign: + andcc %o5, 3, %o3 ! is sp1 aligned on a word boundary + bz,pn %xcc, .wrword + andn %o2, 3, %o3 ! create word sized count in %o3 + + dec %o2 ! decrement count + stb %o1, [%o5] ! clear a byte + b .wdalign + inc %o5 ! next byte + +.wrword: + subcc %o3, 4, %o3 + st %o1, [%o5] ! 4-byte writing loop + bnz,pt %xcc, .wrword + add %o5, 4, %o5 + + and %o2, 3, %o2 ! leftover count, if any + +.wrchar: + ! Set the remaining bytes, if any + brz %o2, .exit + nop +.wrfin: + deccc %o2 + stb %o1, [%o5] + bgu,pt %xcc, .wrfin + inc %o5 +.exit: + retl ! %o0 was preserved + nop + + .size M7memset,.-M7memset diff --git a/arch/sparc/lib/M7patch.S b/arch/sparc/lib/M7patch.S new file mode 100644 index 000000000000..9000b7bc5f2b --- /dev/null +++ b/arch/sparc/lib/M7patch.S @@ -0,0 +1,51 @@ +/* + * M7patch.S: Patch generic routines with M7 variant. + * + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + */ + +#include <linux/linkage.h> + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define NG_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + +ENTRY(m7_patch_copyops) + NG_DO_PATCH(memcpy, M7memcpy) + NG_DO_PATCH(raw_copy_from_user, M7copy_from_user) + NG_DO_PATCH(raw_copy_to_user, M7copy_to_user) + retl + nop +ENDPROC(m7_patch_copyops) + +ENTRY(m7_patch_bzero) + NG_DO_PATCH(memset, M7memset) + NG_DO_PATCH(__bzero, M7bzero) + NG_DO_PATCH(__clear_user, NGclear_user) + NG_DO_PATCH(tsb_init, NGtsb_init) + retl + nop +ENDPROC(m7_patch_bzero) + +ENTRY(m7_patch_pageops) + NG_DO_PATCH(copy_user_page, NG4copy_user_page) + NG_DO_PATCH(_clear_page, M7clear_page) + NG_DO_PATCH(clear_user_page, M7clear_user_page) + retl + nop +ENDPROC(m7_patch_pageops) diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index dbe119b63b48..783bdec0d7be 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -1,8 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 # Makefile for Sparc library files.. # -asflags-y := -ansi -DST_DIV0=0x02 -ccflags-y := -Werror +asflags-y := -DST_DIV0=0x02 lib-$(CONFIG_SPARC32) += ashrdi3.o lib-$(CONFIG_SPARC32) += memcpy.o memset.o @@ -14,7 +14,11 @@ lib-$(CONFIG_SPARC32) += divdi3.o udivdi3.o lib-$(CONFIG_SPARC32) += copy_user.o locks.o lib-$(CONFIG_SPARC64) += atomic_64.o lib-$(CONFIG_SPARC32) += lshrdi3.o ashldi3.o -lib-$(CONFIG_SPARC32) += muldi3.o bitext.o cmpdi2.o +lib-$(CONFIG_SPARC32) += muldi3.o bitext.o +lib-$(CONFIG_SPARC64) += multi3.o +lib-$(CONFIG_SPARC64) += fls.o +lib-$(CONFIG_SPARC64) += fls64.o +lib-$(CONFIG_SPARC64) += NG4fls.o lib-$(CONFIG_SPARC64) += copy_page.o clear_page.o bzero.o lib-$(CONFIG_SPARC64) += csum_copy.o csum_copy_from_user.o csum_copy_to_user.o @@ -35,13 +39,17 @@ lib-$(CONFIG_SPARC64) += NG2patch.o lib-$(CONFIG_SPARC64) += NG4memcpy.o NG4copy_from_user.o NG4copy_to_user.o lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o +lib-$(CONFIG_SPARC64) += Memcpy_utils.o + +lib-$(CONFIG_SPARC64) += M7memcpy.o M7copy_from_user.o M7copy_to_user.o +lib-$(CONFIG_SPARC64) += M7patch.o M7memset.o + lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o -lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o +lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o -obj-y += iomap.o -obj-$(CONFIG_SPARC32) += atomic32.o ucmpdi2.o -obj-y += ksyms.o +obj-$(CONFIG_SPARC64) += iomap.o +obj-$(CONFIG_SPARC32) += atomic32.o obj-$(CONFIG_SPARC64) += PeeCeeI.o diff --git a/arch/sparc/lib/Memcpy_utils.S b/arch/sparc/lib/Memcpy_utils.S new file mode 100644 index 000000000000..207343367bb2 --- /dev/null +++ b/arch/sparc/lib/Memcpy_utils.S @@ -0,0 +1,354 @@ +#ifndef __ASM_MEMCPY_UTILS +#define __ASM_MEMCPY_UTILS + +#include <linux/linkage.h> +#include <asm/asi.h> +#include <asm/visasm.h> + +ENTRY(__restore_asi_fp) + VISExitHalf + retl + wr %g0, ASI_AIUS, %asi +ENDPROC(__restore_asi_fp) + +ENTRY(__restore_asi) + retl + wr %g0, ASI_AIUS, %asi +ENDPROC(__restore_asi) + +ENTRY(memcpy_retl_o2) + ba,pt %xcc, __restore_asi + mov %o2, %o0 +ENDPROC(memcpy_retl_o2) +ENTRY(memcpy_retl_o2_plus_1) + ba,pt %xcc, __restore_asi + add %o2, 1, %o0 +ENDPROC(memcpy_retl_o2_plus_1) +ENTRY(memcpy_retl_o2_plus_3) + ba,pt %xcc, __restore_asi + add %o2, 3, %o0 +ENDPROC(memcpy_retl_o2_plus_3) +ENTRY(memcpy_retl_o2_plus_4) + ba,pt %xcc, __restore_asi + add %o2, 4, %o0 +ENDPROC(memcpy_retl_o2_plus_4) +ENTRY(memcpy_retl_o2_plus_5) + ba,pt %xcc, __restore_asi + add %o2, 5, %o0 +ENDPROC(memcpy_retl_o2_plus_5) +ENTRY(memcpy_retl_o2_plus_6) + ba,pt %xcc, __restore_asi + add %o2, 6, %o0 +ENDPROC(memcpy_retl_o2_plus_6) +ENTRY(memcpy_retl_o2_plus_7) + ba,pt %xcc, __restore_asi + add %o2, 7, %o0 +ENDPROC(memcpy_retl_o2_plus_7) +ENTRY(memcpy_retl_o2_plus_8) + ba,pt %xcc, __restore_asi + add %o2, 8, %o0 +ENDPROC(memcpy_retl_o2_plus_8) +ENTRY(memcpy_retl_o2_plus_15) + ba,pt %xcc, __restore_asi + add %o2, 15, %o0 +ENDPROC(memcpy_retl_o2_plus_15) +ENTRY(memcpy_retl_o2_plus_15_8) + add %o2, 15, %o2 + ba,pt %xcc, __restore_asi + add %o2, 8, %o0 +ENDPROC(memcpy_retl_o2_plus_15_8) +ENTRY(memcpy_retl_o2_plus_16) + ba,pt %xcc, __restore_asi + add %o2, 16, %o0 +ENDPROC(memcpy_retl_o2_plus_16) +ENTRY(memcpy_retl_o2_plus_24) + ba,pt %xcc, __restore_asi + add %o2, 24, %o0 +ENDPROC(memcpy_retl_o2_plus_24) +ENTRY(memcpy_retl_o2_plus_31) + ba,pt %xcc, __restore_asi + add %o2, 31, %o0 +ENDPROC(memcpy_retl_o2_plus_31) +ENTRY(memcpy_retl_o2_plus_32) + ba,pt %xcc, __restore_asi + add %o2, 32, %o0 +ENDPROC(memcpy_retl_o2_plus_32) +ENTRY(memcpy_retl_o2_plus_31_32) + add %o2, 31, %o2 + ba,pt %xcc, __restore_asi + add %o2, 32, %o0 +ENDPROC(memcpy_retl_o2_plus_31_32) +ENTRY(memcpy_retl_o2_plus_31_24) + add %o2, 31, %o2 + ba,pt %xcc, __restore_asi + add %o2, 24, %o0 +ENDPROC(memcpy_retl_o2_plus_31_24) +ENTRY(memcpy_retl_o2_plus_31_16) + add %o2, 31, %o2 + ba,pt %xcc, __restore_asi + add %o2, 16, %o0 +ENDPROC(memcpy_retl_o2_plus_31_16) +ENTRY(memcpy_retl_o2_plus_31_8) + add %o2, 31, %o2 + ba,pt %xcc, __restore_asi + add %o2, 8, %o0 +ENDPROC(memcpy_retl_o2_plus_31_8) +ENTRY(memcpy_retl_o2_plus_63) + ba,pt %xcc, __restore_asi + add %o2, 63, %o0 +ENDPROC(memcpy_retl_o2_plus_63) +ENTRY(memcpy_retl_o2_plus_63_64) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 64, %o0 +ENDPROC(memcpy_retl_o2_plus_63_64) +ENTRY(memcpy_retl_o2_plus_63_56) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 56, %o0 +ENDPROC(memcpy_retl_o2_plus_63_56) +ENTRY(memcpy_retl_o2_plus_63_48) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 48, %o0 +ENDPROC(memcpy_retl_o2_plus_63_48) +ENTRY(memcpy_retl_o2_plus_63_40) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 40, %o0 +ENDPROC(memcpy_retl_o2_plus_63_40) +ENTRY(memcpy_retl_o2_plus_63_32) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 32, %o0 +ENDPROC(memcpy_retl_o2_plus_63_32) +ENTRY(memcpy_retl_o2_plus_63_24) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 24, %o0 +ENDPROC(memcpy_retl_o2_plus_63_24) +ENTRY(memcpy_retl_o2_plus_63_16) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 16, %o0 +ENDPROC(memcpy_retl_o2_plus_63_16) +ENTRY(memcpy_retl_o2_plus_63_8) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 8, %o0 +ENDPROC(memcpy_retl_o2_plus_63_8) +ENTRY(memcpy_retl_o2_plus_o3) + ba,pt %xcc, __restore_asi + add %o2, %o3, %o0 +ENDPROC(memcpy_retl_o2_plus_o3) +ENTRY(memcpy_retl_o2_plus_o3_plus_1) + add %o3, 1, %o3 + ba,pt %xcc, __restore_asi + add %o2, %o3, %o0 +ENDPROC(memcpy_retl_o2_plus_o3_plus_1) +ENTRY(memcpy_retl_o2_plus_o5) + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5) +ENTRY(memcpy_retl_o2_plus_o5_plus_1) + add %o5, 1, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_1) +ENTRY(memcpy_retl_o2_plus_o5_plus_4) + add %o5, 4, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_4) +ENTRY(memcpy_retl_o2_plus_o5_plus_8) + add %o5, 8, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_8) +ENTRY(memcpy_retl_o2_plus_o5_plus_16) + add %o5, 16, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_16) +ENTRY(memcpy_retl_o2_plus_o5_plus_24) + add %o5, 24, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_24) +ENTRY(memcpy_retl_o2_plus_o5_plus_32) + add %o5, 32, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_32) +ENTRY(memcpy_retl_o2_plus_o5_64) + add %o5, 32, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_64) +ENTRY(memcpy_retl_o2_plus_g1) + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(memcpy_retl_o2_plus_g1) +ENTRY(memcpy_retl_o2_plus_g1_plus_1) + add %g1, 1, %g1 + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(memcpy_retl_o2_plus_g1_plus_1) +ENTRY(memcpy_retl_o2_plus_g1_plus_8) + add %g1, 8, %g1 + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(memcpy_retl_o2_plus_g1_plus_8) +ENTRY(memcpy_retl_o2_plus_o4) + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4) +ENTRY(memcpy_retl_o2_plus_o4_plus_8) + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_8) +ENTRY(memcpy_retl_o2_plus_o4_plus_16) + add %o4, 16, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_16) +ENTRY(memcpy_retl_o2_plus_o4_plus_24) + add %o4, 24, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_24) +ENTRY(memcpy_retl_o2_plus_o4_plus_32) + add %o4, 32, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_32) +ENTRY(memcpy_retl_o2_plus_o4_plus_40) + add %o4, 40, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_40) +ENTRY(memcpy_retl_o2_plus_o4_plus_48) + add %o4, 48, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_48) +ENTRY(memcpy_retl_o2_plus_o4_plus_56) + add %o4, 56, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_56) +ENTRY(memcpy_retl_o2_plus_o4_plus_64) + add %o4, 64, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_64) +ENTRY(memcpy_retl_o2_plus_o5_plus_64) + add %o5, 64, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_64) +ENTRY(memcpy_retl_o2_plus_o3_fp) + ba,pt %xcc, __restore_asi_fp + add %o2, %o3, %o0 +ENDPROC(memcpy_retl_o2_plus_o3_fp) +ENTRY(memcpy_retl_o2_plus_o3_plus_1_fp) + add %o3, 1, %o3 + ba,pt %xcc, __restore_asi_fp + add %o2, %o3, %o0 +ENDPROC(memcpy_retl_o2_plus_o3_plus_1_fp) +ENTRY(memcpy_retl_o2_plus_o3_plus_4_fp) + add %o3, 4, %o3 + ba,pt %xcc, __restore_asi_fp + add %o2, %o3, %o0 +ENDPROC(memcpy_retl_o2_plus_o3_plus_4_fp) +ENTRY(memcpy_retl_o2_plus_o4_fp) + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_8_fp) + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_8_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_16_fp) + add %o4, 16, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_16_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_24_fp) + add %o4, 24, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_24_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_32_fp) + add %o4, 32, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_32_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_40_fp) + add %o4, 40, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_40_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_48_fp) + add %o4, 48, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_48_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_56_fp) + add %o4, 56, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_56_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_64_fp) + add %o4, 64, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_64_fp) +ENTRY(memcpy_retl_o2_plus_o5_fp) + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_64_fp) + add %o5, 64, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_64_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_56_fp) + add %o5, 56, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_56_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_48_fp) + add %o5, 48, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_48_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_40_fp) + add %o5, 40, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_40_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_32_fp) + add %o5, 32, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_32_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_24_fp) + add %o5, 24, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_24_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_16_fp) + add %o5, 16, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_16_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_8_fp) + add %o5, 8, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_8_fp) + +#endif diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S index 119ccb9a54f4..e57bc514f538 100644 --- a/arch/sparc/lib/NG2copy_from_user.S +++ b/arch/sparc/lib/NG2copy_from_user.S @@ -1,13 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NG2copy_from_user.S: Niagara-2 optimized copy from userspace. * * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi;\ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_LD_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y##_fp; \ .text; \ .align 4; @@ -28,7 +37,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S index 7fe1ccefd9d0..367c0bf01518 100644 --- a/arch/sparc/lib/NG2copy_to_user.S +++ b/arch/sparc/lib/NG2copy_to_user.S @@ -1,13 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NG2copy_to_user.S: Niagara-2 optimized copy to userspace. * * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi;\ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_ST_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y##_fp; \ .text; \ .align 4; @@ -37,7 +46,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S index 2c20ad63ddbf..bcb21b3a82f1 100644 --- a/arch/sparc/lib/NG2memcpy.S +++ b/arch/sparc/lib/NG2memcpy.S @@ -1,9 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NG2memcpy.S: Niagara-2 optimized memcpy. * * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ #ifdef __KERNEL__ +#include <linux/linkage.h> #include <asm/visasm.h> #include <asm/asi.h> #define GLOBAL_SPARE %g7 @@ -32,15 +34,17 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x +#endif +#ifndef EX_LD_FP +#define EX_LD_FP(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x +#define EX_ST(x,y) x #endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#ifndef EX_ST_FP +#define EX_ST_FP(x,y) x #endif #ifndef LOAD @@ -134,45 +138,110 @@ fsrc2 %x6, %f12; \ fsrc2 %x7, %f14; #define FREG_LOAD_1(base, x0) \ - EX_LD(LOAD(ldd, base + 0x00, %x0)) + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1) #define FREG_LOAD_2(base, x0, x1) \ - EX_LD(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD(LOAD(ldd, base + 0x08, %x1)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); #define FREG_LOAD_3(base, x0, x1, x2) \ - EX_LD(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD(LOAD(ldd, base + 0x10, %x2)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); #define FREG_LOAD_4(base, x0, x1, x2, x3) \ - EX_LD(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD(LOAD(ldd, base + 0x10, %x2)); \ - EX_LD(LOAD(ldd, base + 0x18, %x3)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); #define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \ - EX_LD(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD(LOAD(ldd, base + 0x10, %x2)); \ - EX_LD(LOAD(ldd, base + 0x18, %x3)); \ - EX_LD(LOAD(ldd, base + 0x20, %x4)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); #define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \ - EX_LD(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD(LOAD(ldd, base + 0x10, %x2)); \ - EX_LD(LOAD(ldd, base + 0x18, %x3)); \ - EX_LD(LOAD(ldd, base + 0x20, %x4)); \ - EX_LD(LOAD(ldd, base + 0x28, %x5)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); #define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \ - EX_LD(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD(LOAD(ldd, base + 0x10, %x2)); \ - EX_LD(LOAD(ldd, base + 0x18, %x3)); \ - EX_LD(LOAD(ldd, base + 0x20, %x4)); \ - EX_LD(LOAD(ldd, base + 0x28, %x5)); \ - EX_LD(LOAD(ldd, base + 0x30, %x6)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x30, %x6), NG2_retl_o2_plus_g1); .register %g2,#scratch .register %g3,#scratch .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_fp: + VISExitHalf +__restore_asi: + retl + wr %g0, ASI_AIUS, %asi +ENTRY(NG2_retl_o2) + ba,pt %xcc, __restore_asi + mov %o2, %o0 +ENDPROC(NG2_retl_o2) +ENTRY(NG2_retl_o2_plus_1) + ba,pt %xcc, __restore_asi + add %o2, 1, %o0 +ENDPROC(NG2_retl_o2_plus_1) +ENTRY(NG2_retl_o2_plus_4) + ba,pt %xcc, __restore_asi + add %o2, 4, %o0 +ENDPROC(NG2_retl_o2_plus_4) +ENTRY(NG2_retl_o2_plus_8) + ba,pt %xcc, __restore_asi + add %o2, 8, %o0 +ENDPROC(NG2_retl_o2_plus_8) +ENTRY(NG2_retl_o2_plus_o4_plus_1) + add %o4, 1, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_plus_o4_plus_1) +ENTRY(NG2_retl_o2_plus_o4_plus_8) + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_plus_o4_plus_8) +ENTRY(NG2_retl_o2_plus_o4_plus_16) + add %o4, 16, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_plus_o4_plus_16) +ENTRY(NG2_retl_o2_plus_g1_fp) + ba,pt %xcc, __restore_fp + add %o2, %g1, %o0 +ENDPROC(NG2_retl_o2_plus_g1_fp) +ENTRY(NG2_retl_o2_plus_g1_plus_64_fp) + add %g1, 64, %g1 + ba,pt %xcc, __restore_fp + add %o2, %g1, %o0 +ENDPROC(NG2_retl_o2_plus_g1_plus_64_fp) +ENTRY(NG2_retl_o2_plus_g1_plus_1) + add %g1, 1, %g1 + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(NG2_retl_o2_plus_g1_plus_1) +ENTRY(NG2_retl_o2_and_7_plus_o4) + and %o2, 7, %o2 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_and_7_plus_o4) +ENTRY(NG2_retl_o2_and_7_plus_o4_plus_8) + and %o2, 7, %o2 + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_and_7_plus_o4_plus_8) +#endif + .align 64 .globl FUNC_NAME @@ -224,8 +293,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %g0, %o4, %o4 ! bytes to align dst sub %o2, %o4, %o2 1: subcc %o4, 1, %o4 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o0)) + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_o4_plus_1) + EX_ST(STORE(stb, %g1, %o0), NG2_retl_o2_plus_o4_plus_1) add %o1, 1, %o1 bne,pt %XCC, 1b add %o0, 1, %o0 @@ -236,6 +305,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ */ VISEntryHalf + membar #Sync alignaddr %o1, %g0, %g0 add %o1, (64 - 1), %o4 @@ -257,11 +327,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ blu 170f nop ba,a,pt %xcc, 180f + nop 4: /* 32 <= low bits < 48 */ blu 150f nop ba,a,pt %xcc, 160f + nop 5: /* 0 < low bits < 32 */ blu,a 6f cmp %g2, 8 @@ -269,16 +341,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ blu 130f nop ba,a,pt %xcc, 140f + nop 6: /* 0 < low bits < 16 */ bgeu 120f nop /* fall through for 0 < low bits < 8 */ 110: sub %o4, 64, %g2 - EX_LD(LOAD_BLK(%g2, %f0)) -1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) - EX_LD(LOAD_BLK(%o4, %f16)) + EX_LD_FP(LOAD_BLK(%g2, %f0), NG2_retl_o2_plus_g1) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16) - EX_ST(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -289,10 +362,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 120: sub %o4, 56, %g2 FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12) -1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) - EX_LD(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18) - EX_ST(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -303,10 +376,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 130: sub %o4, 48, %g2 FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10) -1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) - EX_LD(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20) - EX_ST(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_6(f20, f22, f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -317,10 +390,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 140: sub %o4, 40, %g2 FREG_LOAD_5(%g2, f0, f2, f4, f6, f8) -1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) - EX_LD(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22) - EX_ST(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_5(f22, f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -331,10 +404,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 150: sub %o4, 32, %g2 FREG_LOAD_4(%g2, f0, f2, f4, f6) -1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) - EX_LD(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24) - EX_ST(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_4(f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -345,10 +418,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 160: sub %o4, 24, %g2 FREG_LOAD_3(%g2, f0, f2, f4) -1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) - EX_LD(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26) - EX_ST(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_3(f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -359,10 +432,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 170: sub %o4, 16, %g2 FREG_LOAD_2(%g2, f0, f2) -1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) - EX_LD(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28) - EX_ST(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_2(f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -373,10 +446,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 180: sub %o4, 8, %g2 FREG_LOAD_1(%g2, f0) -1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) - EX_LD(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30) - EX_ST(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_1(f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -386,10 +459,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ nop 190: -1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) subcc %g1, 64, %g1 - EX_LD(LOAD_BLK(%o4, %f0)) - EX_ST(STORE_BLK(%f0, %o4 + %g3)) + EX_LD_FP(LOAD_BLK(%o4, %f0), NG2_retl_o2_plus_g1_plus_64) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1_plus_64) add %o4, 64, %o4 bne,pt %xcc, 1b LOAD(prefetch, %o4 + 64, #one_read) @@ -406,6 +479,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ brz,pt %o2, 85f sub %o0, %o1, GLOBAL_SPARE ba,a,pt %XCC, 90f + nop .align 64 75: /* 16 < len <= 64 */ @@ -416,28 +490,28 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0xf, %o4 and %o2, 0xf, %o2 1: subcc %o4, 0x10, %o4 - EX_LD(LOAD(ldx, %o1, %o5)) + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_o4_plus_16) add %o1, 0x08, %o1 - EX_LD(LOAD(ldx, %o1, %g1)) + EX_LD(LOAD(ldx, %o1, %g1), NG2_retl_o2_plus_o4_plus_16) sub %o1, 0x08, %o1 - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_16) add %o1, 0x8, %o1 - EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE)) + EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_8) bgu,pt %XCC, 1b add %o1, 0x8, %o1 73: andcc %o2, 0x8, %g0 be,pt %XCC, 1f nop sub %o2, 0x8, %o2 - EX_LD(LOAD(ldx, %o1, %o5)) - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_8) + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_8) add %o1, 0x8, %o1 1: andcc %o2, 0x4, %g0 be,pt %XCC, 1f nop sub %o2, 0x4, %o2 - EX_LD(LOAD(lduw, %o1, %o5)) - EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(lduw, %o1, %o5), NG2_retl_o2_plus_4) + EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4) add %o1, 0x4, %o1 1: cmp %o2, 0 be,pt %XCC, 85f @@ -453,8 +527,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %o2, %g1, %o2 1: subcc %g1, 1, %g1 - EX_LD(LOAD(ldub, %o1, %o5)) - EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(ldub, %o1, %o5), NG2_retl_o2_plus_g1_plus_1) + EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_g1_plus_1) bgu,pt %icc, 1b add %o1, 1, %o1 @@ -470,16 +544,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 8: mov 64, GLOBAL_SPARE andn %o1, 0x7, %o1 - EX_LD(LOAD(ldx, %o1, %g2)) + EX_LD(LOAD(ldx, %o1, %g2), NG2_retl_o2) sub GLOBAL_SPARE, %g1, GLOBAL_SPARE andn %o2, 0x7, %o4 sllx %g2, %g1, %g2 1: add %o1, 0x8, %o1 - EX_LD(LOAD(ldx, %o1, %g3)) + EX_LD(LOAD(ldx, %o1, %g3), NG2_retl_o2_and_7_plus_o4) subcc %o4, 0x8, %o4 srlx %g3, GLOBAL_SPARE, %o5 or %o5, %g2, %o5 - EX_ST(STORE(stx, %o5, %o0)) + EX_ST(STORE(stx, %o5, %o0), NG2_retl_o2_and_7_plus_o4_plus_8) add %o0, 0x8, %o0 bgu,pt %icc, 1b sllx %g3, %g1, %g2 @@ -499,8 +573,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 1: subcc %o2, 4, %o2 - EX_LD(LOAD(lduw, %o1, %g1)) - EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(lduw, %o1, %g1), NG2_retl_o2_plus_4) + EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4) bgu,pt %XCC, 1b add %o1, 4, %o1 @@ -510,8 +584,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 32 90: subcc %o2, 1, %o2 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_1) + EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_1) bgu,pt %XCC, 90b add %o1, 1, %o1 retl diff --git a/arch/sparc/lib/NG2patch.S b/arch/sparc/lib/NG2patch.S index 28c36f06a6d1..72431b24491a 100644 --- a/arch/sparc/lib/NG2patch.S +++ b/arch/sparc/lib/NG2patch.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NG2patch.S: Patch Ultra-I routines with Niagara-2 variant. * * Copyright (C) 2007 David S. Miller <davem@davemloft.net> @@ -26,8 +27,8 @@ .type niagara2_patch_copyops,#function niagara2_patch_copyops: NG_DO_PATCH(memcpy, NG2memcpy) - NG_DO_PATCH(___copy_from_user, NG2copy_from_user) - NG_DO_PATCH(___copy_to_user, NG2copy_to_user) + NG_DO_PATCH(raw_copy_from_user, NG2copy_from_user) + NG_DO_PATCH(raw_copy_to_user, NG2copy_to_user) retl nop .size niagara2_patch_copyops,.-niagara2_patch_copyops diff --git a/arch/sparc/lib/NG4clear_page.S b/arch/sparc/lib/NG4clear_page.S index e16c88204a42..d91d6b5f2444 100644 --- a/arch/sparc/lib/NG4clear_page.S +++ b/arch/sparc/lib/NG4clear_page.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NG4copy_page.S: Niagara-4 optimized clear page. * * Copyright (C) 2012 (davem@davemloft.net) @@ -26,4 +27,4 @@ NG4clear_user_page: /* %o0=dest, %o1=vaddr */ retl nop .size NG4clear_page,.-NG4clear_page - .size NG4clear_user_page,.-NG4clear_user_page
\ No newline at end of file + .size NG4clear_user_page,.-NG4clear_user_page diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S index fd9f903ffa32..0cac15a6db3c 100644 --- a/arch/sparc/lib/NG4copy_from_user.S +++ b/arch/sparc/lib/NG4copy_from_user.S @@ -1,13 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NG4copy_from_user.S: Niagara-4 optimized copy from userspace. * * Copyright (C) 2012 David S. Miller (davem@davemloft.net) */ -#define EX_LD(x) \ +#define EX_LD(x, y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi;\ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_LD_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y##_fp; \ .text; \ .align 4; @@ -23,7 +32,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NG4copy_page.S b/arch/sparc/lib/NG4copy_page.S index 28504e88c535..581062f8ba5f 100644 --- a/arch/sparc/lib/NG4copy_page.S +++ b/arch/sparc/lib/NG4copy_page.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NG4copy_page.S: Niagara-4 optimized copy page. * * Copyright (C) 2012 (davem@davemloft.net) diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S index 9744c4540a8d..c5c9abb3cb77 100644 --- a/arch/sparc/lib/NG4copy_to_user.S +++ b/arch/sparc/lib/NG4copy_to_user.S @@ -1,13 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NG4copy_to_user.S: Niagara-4 optimized copy to userspace. * * Copyright (C) 2012 David S. Miller (davem@davemloft.net) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi;\ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_ST_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y##_fp; \ .text; \ .align 4; @@ -32,7 +41,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NG4fls.S b/arch/sparc/lib/NG4fls.S new file mode 100644 index 000000000000..2d0991e5b034 --- /dev/null +++ b/arch/sparc/lib/NG4fls.S @@ -0,0 +1,30 @@ +/* NG4fls.S: SPARC optimized fls and __fls for T4 and above. + * + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + */ + +#include <linux/linkage.h> + +#define LZCNT_O0_G2 \ + .word 0x85b002e8 + + .text + .register %g2, #scratch + .register %g3, #scratch + +ENTRY(NG4fls) + LZCNT_O0_G2 !lzcnt %o0, %g2 + mov 64, %g3 + retl + sub %g3, %g2, %o0 +ENDPROC(NG4fls) + +ENTRY(__NG4fls) + brz,pn %o0, 1f + LZCNT_O0_G2 !lzcnt %o0, %g2 + mov 63, %g3 + sub %g3, %g2, %o0 +1: + retl + nop +ENDPROC(__NG4fls) diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S index 9cf2ee01cee3..df0ec1bd1948 100644 --- a/arch/sparc/lib/NG4memcpy.S +++ b/arch/sparc/lib/NG4memcpy.S @@ -1,9 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NG4memcpy.S: Niagara-4 optimized memcpy. * * Copyright (C) 2012 David S. Miller (davem@davemloft.net) */ #ifdef __KERNEL__ +#include <linux/linkage.h> #include <asm/visasm.h> #include <asm/asi.h> #define GLOBAL_SPARE %g7 @@ -41,18 +43,25 @@ #endif #endif +#if !defined(EX_LD) && !defined(EX_ST) +#define NON_USER_COPY +#endif + #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x +#endif +#ifndef EX_LD_FP +#define EX_LD_FP(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x +#define EX_ST(x,y) x #endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#ifndef EX_ST_FP +#define EX_ST_FP(x,y) x #endif + #ifndef LOAD #define LOAD(type,addr,dest) type [addr], dest #endif @@ -84,6 +93,9 @@ .register %g3,#scratch .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +#endif .align 64 .globl FUNC_NAME @@ -114,12 +126,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ brz,pt %g1, 51f sub %o2, %g1, %o2 -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2)) + +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), memcpy_retl_o2_plus_g1) add %o1, 1, %o1 subcc %g1, 1, %g1 add %o0, 1, %o0 bne,pt %icc, 1b - EX_ST(STORE(stb, %g2, %o0 - 0x01)) + EX_ST(STORE(stb, %g2, %o0 - 0x01), memcpy_retl_o2_plus_g1_plus_1) 51: LOAD(prefetch, %o1 + 0x040, #n_reads_strong) LOAD(prefetch, %o1 + 0x080, #n_reads_strong) @@ -144,43 +157,43 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ brz,pt %g1, .Llarge_aligned sub %o2, %g1, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2)) +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2), memcpy_retl_o2_plus_g1) add %o1, 8, %o1 subcc %g1, 8, %g1 add %o0, 8, %o0 bne,pt %icc, 1b - EX_ST(STORE(stx, %g2, %o0 - 0x08)) + EX_ST(STORE(stx, %g2, %o0 - 0x08), memcpy_retl_o2_plus_g1_plus_8) .Llarge_aligned: /* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */ andn %o2, 0x3f, %o4 sub %o2, %o4, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), memcpy_retl_o2_plus_o4) add %o1, 0x40, %o1 - EX_LD(LOAD(ldx, %o1 - 0x38, %g2)) + EX_LD(LOAD(ldx, %o1 - 0x38, %g2), memcpy_retl_o2_plus_o4) subcc %o4, 0x40, %o4 - EX_LD(LOAD(ldx, %o1 - 0x30, %g3)) - EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE)) - EX_LD(LOAD(ldx, %o1 - 0x20, %o5)) - EX_ST(STORE_INIT(%g1, %o0)) + EX_LD(LOAD(ldx, %o1 - 0x30, %g3), memcpy_retl_o2_plus_o4_plus_64) + EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE), memcpy_retl_o2_plus_o4_plus_64) + EX_LD(LOAD(ldx, %o1 - 0x20, %o5), memcpy_retl_o2_plus_o4_plus_64) + EX_ST(STORE_INIT(%g1, %o0), memcpy_retl_o2_plus_o4_plus_64) add %o0, 0x08, %o0 - EX_ST(STORE_INIT(%g2, %o0)) + EX_ST(STORE_INIT(%g2, %o0), memcpy_retl_o2_plus_o4_plus_56) add %o0, 0x08, %o0 - EX_LD(LOAD(ldx, %o1 - 0x18, %g2)) - EX_ST(STORE_INIT(%g3, %o0)) + EX_LD(LOAD(ldx, %o1 - 0x18, %g2), memcpy_retl_o2_plus_o4_plus_48) + EX_ST(STORE_INIT(%g3, %o0), memcpy_retl_o2_plus_o4_plus_48) add %o0, 0x08, %o0 - EX_LD(LOAD(ldx, %o1 - 0x10, %g3)) - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0)) + EX_LD(LOAD(ldx, %o1 - 0x10, %g3), memcpy_retl_o2_plus_o4_plus_40) + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), memcpy_retl_o2_plus_o4_plus_40) add %o0, 0x08, %o0 - EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE)) - EX_ST(STORE_INIT(%o5, %o0)) + EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE), memcpy_retl_o2_plus_o4_plus_32) + EX_ST(STORE_INIT(%o5, %o0), memcpy_retl_o2_plus_o4_plus_32) add %o0, 0x08, %o0 - EX_ST(STORE_INIT(%g2, %o0)) + EX_ST(STORE_INIT(%g2, %o0), memcpy_retl_o2_plus_o4_plus_24) add %o0, 0x08, %o0 - EX_ST(STORE_INIT(%g3, %o0)) + EX_ST(STORE_INIT(%g3, %o0), memcpy_retl_o2_plus_o4_plus_16) add %o0, 0x08, %o0 - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0)) + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), memcpy_retl_o2_plus_o4_plus_8) add %o0, 0x08, %o0 bne,pt %icc, 1b LOAD(prefetch, %o1 + 0x200, #n_reads_strong) @@ -197,22 +210,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ mov EX_RETVAL(%o3), %o0 .Llarge_src_unaligned: +#ifdef NON_USER_COPY + VISEntryHalfFast(.Lmedium_vis_entry_fail) +#else + VISEntryHalf +#endif andn %o2, 0x3f, %o4 sub %o2, %o4, %o2 - VISEntryHalf alignaddr %o1, %g0, %g1 add %o1, %o4, %o1 - EX_LD(LOAD(ldd, %g1 + 0x00, %f0)) -1: EX_LD(LOAD(ldd, %g1 + 0x08, %f2)) + EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0), memcpy_retl_o2_plus_o4) +1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2), memcpy_retl_o2_plus_o4) subcc %o4, 0x40, %o4 - EX_LD(LOAD(ldd, %g1 + 0x10, %f4)) - EX_LD(LOAD(ldd, %g1 + 0x18, %f6)) - EX_LD(LOAD(ldd, %g1 + 0x20, %f8)) - EX_LD(LOAD(ldd, %g1 + 0x28, %f10)) - EX_LD(LOAD(ldd, %g1 + 0x30, %f12)) - EX_LD(LOAD(ldd, %g1 + 0x38, %f14)) + EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4), memcpy_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6), memcpy_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8), memcpy_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10), memcpy_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12), memcpy_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14), memcpy_retl_o2_plus_o4_plus_64) faligndata %f0, %f2, %f16 - EX_LD(LOAD(ldd, %g1 + 0x40, %f0)) + EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0), memcpy_retl_o2_plus_o4_plus_64) faligndata %f2, %f4, %f18 add %g1, 0x40, %g1 faligndata %f4, %f6, %f20 @@ -221,25 +238,32 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ faligndata %f10, %f12, %f26 faligndata %f12, %f14, %f28 faligndata %f14, %f0, %f30 - EX_ST(STORE(std, %f16, %o0 + 0x00)) - EX_ST(STORE(std, %f18, %o0 + 0x08)) - EX_ST(STORE(std, %f20, %o0 + 0x10)) - EX_ST(STORE(std, %f22, %o0 + 0x18)) - EX_ST(STORE(std, %f24, %o0 + 0x20)) - EX_ST(STORE(std, %f26, %o0 + 0x28)) - EX_ST(STORE(std, %f28, %o0 + 0x30)) - EX_ST(STORE(std, %f30, %o0 + 0x38)) + EX_ST_FP(STORE(std, %f16, %o0 + 0x00), memcpy_retl_o2_plus_o4_plus_64) + EX_ST_FP(STORE(std, %f18, %o0 + 0x08), memcpy_retl_o2_plus_o4_plus_56) + EX_ST_FP(STORE(std, %f20, %o0 + 0x10), memcpy_retl_o2_plus_o4_plus_48) + EX_ST_FP(STORE(std, %f22, %o0 + 0x18), memcpy_retl_o2_plus_o4_plus_40) + EX_ST_FP(STORE(std, %f24, %o0 + 0x20), memcpy_retl_o2_plus_o4_plus_32) + EX_ST_FP(STORE(std, %f26, %o0 + 0x28), memcpy_retl_o2_plus_o4_plus_24) + EX_ST_FP(STORE(std, %f28, %o0 + 0x30), memcpy_retl_o2_plus_o4_plus_16) + EX_ST_FP(STORE(std, %f30, %o0 + 0x38), memcpy_retl_o2_plus_o4_plus_8) add %o0, 0x40, %o0 bne,pt %icc, 1b LOAD(prefetch, %g1 + 0x200, #n_reads_strong) +#ifdef NON_USER_COPY + VISExitHalfFast +#else VISExitHalf - +#endif brz,pn %o2, .Lexit cmp %o2, 19 ble,pn %icc, .Lsmall_unaligned nop ba,a,pt %icc, .Lmedium_unaligned +#ifdef NON_USER_COPY +.Lmedium_vis_entry_fail: + or %o0, %o1, %g2 +#endif .Lmedium: LOAD(prefetch, %o1 + 0x40, #n_reads_strong) andcc %g2, 0x7, %g0 @@ -249,37 +273,38 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andncc %o2, 0x20 - 1, %o5 be,pn %icc, 2f sub %o2, %o5, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) - EX_LD(LOAD(ldx, %o1 + 0x08, %g2)) - EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE)) - EX_LD(LOAD(ldx, %o1 + 0x18, %o4)) +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x08, %g2), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x18, %o4), memcpy_retl_o2_plus_o5) add %o1, 0x20, %o1 subcc %o5, 0x20, %o5 - EX_ST(STORE(stx, %g1, %o0 + 0x00)) - EX_ST(STORE(stx, %g2, %o0 + 0x08)) - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10)) - EX_ST(STORE(stx, %o4, %o0 + 0x18)) + EX_ST(STORE(stx, %g1, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_32) + EX_ST(STORE(stx, %g2, %o0 + 0x08), memcpy_retl_o2_plus_o5_plus_24) + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), memcpy_retl_o2_plus_o5_plus_16) + EX_ST(STORE(stx, %o4, %o0 + 0x18), memcpy_retl_o2_plus_o5_plus_8) bne,pt %icc, 1b add %o0, 0x20, %o0 2: andcc %o2, 0x18, %o5 be,pt %icc, 3f sub %o2, %o5, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) + +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), memcpy_retl_o2_plus_o5) add %o1, 0x08, %o1 add %o0, 0x08, %o0 subcc %o5, 0x08, %o5 bne,pt %icc, 1b - EX_ST(STORE(stx, %g1, %o0 - 0x08)) + EX_ST(STORE(stx, %g1, %o0 - 0x08), memcpy_retl_o2_plus_o5_plus_8) 3: brz,pt %o2, .Lexit cmp %o2, 0x04 bl,pn %icc, .Ltiny nop - EX_LD(LOAD(lduw, %o1 + 0x00, %g1)) + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), memcpy_retl_o2) add %o1, 0x04, %o1 add %o0, 0x04, %o0 subcc %o2, 0x04, %o2 bne,pn %icc, .Ltiny - EX_ST(STORE(stw, %g1, %o0 - 0x04)) + EX_ST(STORE(stw, %g1, %o0 - 0x04), memcpy_retl_o2_plus_4) ba,a,pt %icc, .Lexit .Lmedium_unaligned: /* First get dest 8 byte aligned. */ @@ -288,12 +313,12 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ brz,pt %g1, 2f sub %o2, %g1, %o2 -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2)) +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), memcpy_retl_o2_plus_g1) add %o1, 1, %o1 subcc %g1, 1, %g1 add %o0, 1, %o0 bne,pt %icc, 1b - EX_ST(STORE(stb, %g2, %o0 - 0x01)) + EX_ST(STORE(stb, %g2, %o0 - 0x01), memcpy_retl_o2_plus_g1_plus_1) 2: and %o1, 0x7, %g1 brz,pn %g1, .Lmedium_noprefetch @@ -301,16 +326,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ mov 64, %g2 sub %g2, %g1, %g2 andn %o1, 0x7, %o1 - EX_LD(LOAD(ldx, %o1 + 0x00, %o4)) + EX_LD(LOAD(ldx, %o1 + 0x00, %o4), memcpy_retl_o2) sllx %o4, %g1, %o4 andn %o2, 0x08 - 1, %o5 sub %o2, %o5, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3)) +1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), memcpy_retl_o2_plus_o5) add %o1, 0x08, %o1 subcc %o5, 0x08, %o5 srlx %g3, %g2, GLOBAL_SPARE or GLOBAL_SPARE, %o4, GLOBAL_SPARE - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00)) + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_8) add %o0, 0x08, %o0 bne,pt %icc, 1b sllx %g3, %g1, %o4 @@ -321,17 +346,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ ba,pt %icc, .Lsmall_unaligned .Ltiny: - EX_LD(LOAD(ldub, %o1 + 0x00, %g1)) + EX_LD(LOAD(ldub, %o1 + 0x00, %g1), memcpy_retl_o2) subcc %o2, 1, %o2 be,pn %icc, .Lexit - EX_ST(STORE(stb, %g1, %o0 + 0x00)) - EX_LD(LOAD(ldub, %o1 + 0x01, %g1)) + EX_ST(STORE(stb, %g1, %o0 + 0x00), memcpy_retl_o2_plus_1) + EX_LD(LOAD(ldub, %o1 + 0x01, %g1), memcpy_retl_o2) subcc %o2, 1, %o2 be,pn %icc, .Lexit - EX_ST(STORE(stb, %g1, %o0 + 0x01)) - EX_LD(LOAD(ldub, %o1 + 0x02, %g1)) + EX_ST(STORE(stb, %g1, %o0 + 0x01), memcpy_retl_o2_plus_1) + EX_LD(LOAD(ldub, %o1 + 0x02, %g1), memcpy_retl_o2) ba,pt %icc, .Lexit - EX_ST(STORE(stb, %g1, %o0 + 0x02)) + EX_ST(STORE(stb, %g1, %o0 + 0x02), memcpy_retl_o2) .Lsmall: andcc %g2, 0x3, %g0 @@ -339,22 +364,23 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0x4 - 1, %o5 sub %o2, %o5, %o2 1: - EX_LD(LOAD(lduw, %o1 + 0x00, %g1)) + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), memcpy_retl_o2_plus_o5) add %o1, 0x04, %o1 subcc %o5, 0x04, %o5 add %o0, 0x04, %o0 bne,pt %icc, 1b - EX_ST(STORE(stw, %g1, %o0 - 0x04)) + EX_ST(STORE(stw, %g1, %o0 - 0x04), memcpy_retl_o2_plus_o5_plus_4) brz,pt %o2, .Lexit nop ba,a,pt %icc, .Ltiny .Lsmall_unaligned: -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1)) +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1), memcpy_retl_o2) add %o1, 1, %o1 add %o0, 1, %o0 subcc %o2, 1, %o2 bne,pt %icc, 1b - EX_ST(STORE(stb, %g1, %o0 - 0x01)) + EX_ST(STORE(stb, %g1, %o0 - 0x01), memcpy_retl_o2_plus_1) ba,a,pt %icc, .Lexit + nop .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/NG4memset.S b/arch/sparc/lib/NG4memset.S index 41da4bdd95cb..f81ee5419e2c 100644 --- a/arch/sparc/lib/NG4memset.S +++ b/arch/sparc/lib/NG4memset.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NG4memset.S: Niagara-4 optimized memset/bzero. * * Copyright (C) 2012 David S. Miller (davem@davemloft.net) @@ -102,4 +103,5 @@ NG4bzero: bne,pt %icc, 1b add %o0, 0x30, %o0 ba,a,pt %icc, .Lpostloop + nop .size NG4bzero,.-NG4bzero diff --git a/arch/sparc/lib/NG4patch.S b/arch/sparc/lib/NG4patch.S index a114cbcf2a48..37866175c921 100644 --- a/arch/sparc/lib/NG4patch.S +++ b/arch/sparc/lib/NG4patch.S @@ -1,8 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NG4patch.S: Patch Ultra-I routines with Niagara-4 variant. * * Copyright (C) 2012 David S. Miller <davem@davemloft.net> */ +#include <linux/linkage.h> + #define BRANCH_ALWAYS 0x10680000 #define NOP 0x01000000 #define NG_DO_PATCH(OLD, NEW) \ @@ -26,8 +29,8 @@ .type niagara4_patch_copyops,#function niagara4_patch_copyops: NG_DO_PATCH(memcpy, NG4memcpy) - NG_DO_PATCH(___copy_from_user, NG4copy_from_user) - NG_DO_PATCH(___copy_to_user, NG4copy_to_user) + NG_DO_PATCH(raw_copy_from_user, NG4copy_from_user) + NG_DO_PATCH(raw_copy_to_user, NG4copy_to_user) retl nop .size niagara4_patch_copyops,.-niagara4_patch_copyops @@ -52,3 +55,10 @@ niagara4_patch_pageops: retl nop .size niagara4_patch_pageops,.-niagara4_patch_pageops + +ENTRY(niagara4_patch_fls) + NG_DO_PATCH(fls, NG4fls) + NG_DO_PATCH(__fls, __NG4fls) + retl + nop +ENDPROC(niagara4_patch_fls) diff --git a/arch/sparc/lib/NGbzero.S b/arch/sparc/lib/NGbzero.S index beab29bf419b..19327614d57d 100644 --- a/arch/sparc/lib/NGbzero.S +++ b/arch/sparc/lib/NGbzero.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NGbzero.S: Niagara optimized memset/clear_user. * * Copyright (C) 2006 David S. Miller (davem@davemloft.net) @@ -8,7 +9,7 @@ 98: x,y; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_o1; \ + .word 98b, __retl_o1_asi;\ .text; \ .align 4; diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S index 5d1e4d1ac21e..9abc49fcdbbe 100644 --- a/arch/sparc/lib/NGcopy_from_user.S +++ b/arch/sparc/lib/NGcopy_from_user.S @@ -1,13 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NGcopy_from_user.S: Niagara optimized copy from userspace. * * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __ret_one_asi;\ + .word 98b, y; \ .text; \ .align 4; @@ -25,7 +26,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S index ff630dcb273c..9cbe2f18e5cc 100644 --- a/arch/sparc/lib/NGcopy_to_user.S +++ b/arch/sparc/lib/NGcopy_to_user.S @@ -1,13 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NGcopy_to_user.S: Niagara optimized copy to userspace. * * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __ret_one_asi;\ + .word 98b, y; \ .text; \ .align 4; @@ -28,7 +29,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop #endif diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S index 96a14caf6966..bbd3ea0a6482 100644 --- a/arch/sparc/lib/NGmemcpy.S +++ b/arch/sparc/lib/NGmemcpy.S @@ -1,15 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NGmemcpy.S: Niagara optimized memcpy. * * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) */ #ifdef __KERNEL__ +#include <linux/linkage.h> #include <asm/asi.h> #include <asm/thread_info.h> #define GLOBAL_SPARE %g7 #define RESTORE_ASI(TMP) \ - ldub [%g6 + TI_CURRENT_DS], TMP; \ - wr TMP, 0x0, %asi; + wr %g0, ASI_AIUS, %asi #else #define GLOBAL_SPARE %g5 #define RESTORE_ASI(TMP) \ @@ -27,15 +28,11 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x -#endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#define EX_ST(x,y) x #endif #ifndef LOAD @@ -79,6 +76,99 @@ .register %g3,#scratch .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_asi: + wr %g0, ASI_AIUS, %asi + ret + restore +ENTRY(NG_ret_i2_plus_i4_plus_1) + ba,pt %xcc, __restore_asi + add %i2, %i5, %i0 +ENDPROC(NG_ret_i2_plus_i4_plus_1) +ENTRY(NG_ret_i2_plus_g1) + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1) +ENTRY(NG_ret_i2_plus_g1_minus_8) + sub %g1, 8, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_8) +ENTRY(NG_ret_i2_plus_g1_minus_16) + sub %g1, 16, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_16) +ENTRY(NG_ret_i2_plus_g1_minus_24) + sub %g1, 24, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_24) +ENTRY(NG_ret_i2_plus_g1_minus_32) + sub %g1, 32, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_32) +ENTRY(NG_ret_i2_plus_g1_minus_40) + sub %g1, 40, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_40) +ENTRY(NG_ret_i2_plus_g1_minus_48) + sub %g1, 48, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_48) +ENTRY(NG_ret_i2_plus_g1_minus_56) + sub %g1, 56, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_56) +ENTRY(NG_ret_i2_plus_i4_plus_16) + add %i4, 16, %i4 + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_plus_i4_plus_16) +ENTRY(NG_ret_i2_plus_i4_plus_8) + add %i4, 8, %i4 + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_plus_i4_plus_8) +ENTRY(NG_ret_i2_plus_8) + ba,pt %xcc, __restore_asi + add %i2, 8, %i0 +ENDPROC(NG_ret_i2_plus_8) +ENTRY(NG_ret_i2_plus_4) + ba,pt %xcc, __restore_asi + add %i2, 4, %i0 +ENDPROC(NG_ret_i2_plus_4) +ENTRY(NG_ret_i2_plus_1) + ba,pt %xcc, __restore_asi + add %i2, 1, %i0 +ENDPROC(NG_ret_i2_plus_1) +ENTRY(NG_ret_i2_plus_g1_plus_1) + add %g1, 1, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_plus_1) +ENTRY(NG_ret_i2) + ba,pt %xcc, __restore_asi + mov %i2, %i0 +ENDPROC(NG_ret_i2) +ENTRY(NG_ret_i2_and_7_plus_i4) + and %i2, 7, %i2 + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_and_7_plus_i4) +ENTRY(NG_ret_i2_and_7_plus_i4_plus_8) + and %i2, 7, %i2 + add %i4, 8, %i4 + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_and_7_plus_i4) +#endif + .align 64 .globl FUNC_NAME @@ -126,8 +216,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ sub %g0, %i4, %i4 ! bytes to align dst sub %i2, %i4, %i2 1: subcc %i4, 1, %i4 - EX_LD(LOAD(ldub, %i1, %g1)) - EX_ST(STORE(stb, %g1, %o0)) + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_i4_plus_1) + EX_ST(STORE(stb, %g1, %o0), NG_ret_i2_plus_i4_plus_1) add %i1, 1, %i1 bne,pt %XCC, 1b add %o0, 1, %o0 @@ -160,7 +250,7 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ and %i4, 0x7, GLOBAL_SPARE sll GLOBAL_SPARE, 3, GLOBAL_SPARE mov 64, %i5 - EX_LD(LOAD_TWIN(%i1, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1, %g2, %g3), NG_ret_i2_plus_g1) sub %i5, GLOBAL_SPARE, %i5 mov 16, %o4 mov 32, %o5 @@ -178,31 +268,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ srlx WORD3, PRE_SHIFT, TMP; \ or WORD2, TMP, WORD2; -8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3)) +8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1) MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) LOAD(prefetch, %i1 + %i3, #one_read) - EX_ST(STORE_INIT(%g2, %o0 + 0x00)) - EX_ST(STORE_INIT(%g3, %o0 + 0x08)) + EX_ST(STORE_INIT(%g2, %o0 + 0x00), NG_ret_i2_plus_g1) + EX_ST(STORE_INIT(%g3, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16) MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%o2, %o0 + 0x10)) - EX_ST(STORE_INIT(%o3, %o0 + 0x18)) + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32) MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%g2, %o0 + 0x20)) - EX_ST(STORE_INIT(%g3, %o0 + 0x28)) + EX_ST(STORE_INIT(%g2, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%g3, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48) add %i1, 64, %i1 MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%o2, %o0 + 0x30)) - EX_ST(STORE_INIT(%o3, %o0 + 0x38)) + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) subcc %g1, 64, %g1 bne,pt %XCC, 8b @@ -211,31 +301,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ ba,pt %XCC, 60f add %i1, %i4, %i1 -9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3)) +9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1) MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) LOAD(prefetch, %i1 + %i3, #one_read) - EX_ST(STORE_INIT(%g3, %o0 + 0x00)) - EX_ST(STORE_INIT(%o2, %o0 + 0x08)) + EX_ST(STORE_INIT(%g3, %o0 + 0x00), NG_ret_i2_plus_g1) + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16) MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%o3, %o0 + 0x10)) - EX_ST(STORE_INIT(%g2, %o0 + 0x18)) + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%g2, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32) MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%g3, %o0 + 0x20)) - EX_ST(STORE_INIT(%o2, %o0 + 0x28)) + EX_ST(STORE_INIT(%g3, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48) add %i1, 64, %i1 MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%o3, %o0 + 0x30)) - EX_ST(STORE_INIT(%g2, %o0 + 0x38)) + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%g2, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) subcc %g1, 64, %g1 bne,pt %XCC, 9b @@ -249,25 +339,25 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ * one twin load ahead, then add 8 back into source when * we finish the loop. */ - EX_LD(LOAD_TWIN(%i1, %o4, %o5)) + EX_LD(LOAD_TWIN(%i1, %o4, %o5), NG_ret_i2_plus_g1) mov 16, %o7 mov 32, %g2 mov 48, %g3 mov 64, %o1 -1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) +1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1) LOAD(prefetch, %i1 + %o1, #one_read) - EX_ST(STORE_INIT(%o5, %o0 + 0x00)) ! initializes cache line - EX_ST(STORE_INIT(%o2, %o0 + 0x08)) - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5)) - EX_ST(STORE_INIT(%o3, %o0 + 0x10)) - EX_ST(STORE_INIT(%o4, %o0 + 0x18)) - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3)) - EX_ST(STORE_INIT(%o5, %o0 + 0x20)) - EX_ST(STORE_INIT(%o2, %o0 + 0x28)) - EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5)) + EX_ST(STORE_INIT(%o5, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o4, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o5, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) + EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5), NG_ret_i2_plus_g1_minus_48) add %i1, 64, %i1 - EX_ST(STORE_INIT(%o3, %o0 + 0x30)) - EX_ST(STORE_INIT(%o4, %o0 + 0x38)) + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%o4, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) subcc %g1, 64, %g1 bne,pt %XCC, 1b add %o0, 64, %o0 @@ -282,20 +372,20 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ mov 32, %g2 mov 48, %g3 mov 64, %o1 -1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5)) - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) +1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5), NG_ret_i2_plus_g1) + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1) LOAD(prefetch, %i1 + %o1, #one_read) - EX_ST(STORE_INIT(%o4, %o0 + 0x00)) ! initializes cache line - EX_ST(STORE_INIT(%o5, %o0 + 0x08)) - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5)) - EX_ST(STORE_INIT(%o2, %o0 + 0x10)) - EX_ST(STORE_INIT(%o3, %o0 + 0x18)) - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3)) + EX_ST(STORE_INIT(%o4, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line + EX_ST(STORE_INIT(%o5, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32) add %i1, 64, %i1 - EX_ST(STORE_INIT(%o4, %o0 + 0x20)) - EX_ST(STORE_INIT(%o5, %o0 + 0x28)) - EX_ST(STORE_INIT(%o2, %o0 + 0x30)) - EX_ST(STORE_INIT(%o3, %o0 + 0x38)) + EX_ST(STORE_INIT(%o4, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o5, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) subcc %g1, 64, %g1 bne,pt %XCC, 1b add %o0, 64, %o0 @@ -311,6 +401,7 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ brz,pt %i2, 85f sub %o0, %i1, %i3 ba,a,pt %XCC, 90f + nop .align 64 70: /* 16 < len <= 64 */ @@ -321,28 +412,28 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ andn %i2, 0xf, %i4 and %i2, 0xf, %i2 1: subcc %i4, 0x10, %i4 - EX_LD(LOAD(ldx, %i1, %o4)) + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_i4_plus_16) add %i1, 0x08, %i1 - EX_LD(LOAD(ldx, %i1, %g1)) + EX_LD(LOAD(ldx, %i1, %g1), NG_ret_i2_plus_i4_plus_16) sub %i1, 0x08, %i1 - EX_ST(STORE(stx, %o4, %i1 + %i3)) + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_i4_plus_16) add %i1, 0x8, %i1 - EX_ST(STORE(stx, %g1, %i1 + %i3)) + EX_ST(STORE(stx, %g1, %i1 + %i3), NG_ret_i2_plus_i4_plus_8) bgu,pt %XCC, 1b add %i1, 0x8, %i1 73: andcc %i2, 0x8, %g0 be,pt %XCC, 1f nop sub %i2, 0x8, %i2 - EX_LD(LOAD(ldx, %i1, %o4)) - EX_ST(STORE(stx, %o4, %i1 + %i3)) + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_8) + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_8) add %i1, 0x8, %i1 1: andcc %i2, 0x4, %g0 be,pt %XCC, 1f nop sub %i2, 0x4, %i2 - EX_LD(LOAD(lduw, %i1, %i5)) - EX_ST(STORE(stw, %i5, %i1 + %i3)) + EX_LD(LOAD(lduw, %i1, %i5), NG_ret_i2_plus_4) + EX_ST(STORE(stw, %i5, %i1 + %i3), NG_ret_i2_plus_4) add %i1, 0x4, %i1 1: cmp %i2, 0 be,pt %XCC, 85f @@ -358,8 +449,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ sub %i2, %g1, %i2 1: subcc %g1, 1, %g1 - EX_LD(LOAD(ldub, %i1, %i5)) - EX_ST(STORE(stb, %i5, %i1 + %i3)) + EX_LD(LOAD(ldub, %i1, %i5), NG_ret_i2_plus_g1_plus_1) + EX_ST(STORE(stb, %i5, %i1 + %i3), NG_ret_i2_plus_g1_plus_1) bgu,pt %icc, 1b add %i1, 1, %i1 @@ -375,16 +466,16 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ 8: mov 64, %i3 andn %i1, 0x7, %i1 - EX_LD(LOAD(ldx, %i1, %g2)) + EX_LD(LOAD(ldx, %i1, %g2), NG_ret_i2) sub %i3, %g1, %i3 andn %i2, 0x7, %i4 sllx %g2, %g1, %g2 1: add %i1, 0x8, %i1 - EX_LD(LOAD(ldx, %i1, %g3)) + EX_LD(LOAD(ldx, %i1, %g3), NG_ret_i2_and_7_plus_i4) subcc %i4, 0x8, %i4 srlx %g3, %i3, %i5 or %i5, %g2, %i5 - EX_ST(STORE(stx, %i5, %o0)) + EX_ST(STORE(stx, %i5, %o0), NG_ret_i2_and_7_plus_i4_plus_8) add %o0, 0x8, %o0 bgu,pt %icc, 1b sllx %g3, %g1, %g2 @@ -404,8 +495,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ 1: subcc %i2, 4, %i2 - EX_LD(LOAD(lduw, %i1, %g1)) - EX_ST(STORE(stw, %g1, %i1 + %i3)) + EX_LD(LOAD(lduw, %i1, %g1), NG_ret_i2_plus_4) + EX_ST(STORE(stw, %g1, %i1 + %i3), NG_ret_i2_plus_4) bgu,pt %XCC, 1b add %i1, 4, %i1 @@ -415,8 +506,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ .align 32 90: subcc %i2, 1, %i2 - EX_LD(LOAD(ldub, %i1, %g1)) - EX_ST(STORE(stb, %g1, %i1 + %i3)) + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_1) + EX_ST(STORE(stb, %g1, %i1 + %i3), NG_ret_i2_plus_1) bgu,pt %XCC, 90b add %i1, 1, %i1 ret diff --git a/arch/sparc/lib/NGpage.S b/arch/sparc/lib/NGpage.S index 423d46e2258b..88fec7818065 100644 --- a/arch/sparc/lib/NGpage.S +++ b/arch/sparc/lib/NGpage.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NGpage.S: Niagara optimize clear and copy page. * * Copyright (C) 2006 (davem@davemloft.net) diff --git a/arch/sparc/lib/NGpatch.S b/arch/sparc/lib/NGpatch.S index 3b0674fc3366..e9f843f1063e 100644 --- a/arch/sparc/lib/NGpatch.S +++ b/arch/sparc/lib/NGpatch.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* NGpatch.S: Patch Ultra-I routines with Niagara variant. * * Copyright (C) 2006 David S. Miller <davem@davemloft.net> @@ -26,8 +27,8 @@ .type niagara_patch_copyops,#function niagara_patch_copyops: NG_DO_PATCH(memcpy, NGmemcpy) - NG_DO_PATCH(___copy_from_user, NGcopy_from_user) - NG_DO_PATCH(___copy_to_user, NGcopy_to_user) + NG_DO_PATCH(raw_copy_from_user, NGcopy_from_user) + NG_DO_PATCH(raw_copy_to_user, NGcopy_to_user) retl nop .size niagara_patch_copyops,.-niagara_patch_copyops diff --git a/arch/sparc/lib/PeeCeeI.c b/arch/sparc/lib/PeeCeeI.c index 6529f8657597..cde4c9a51b2e 100644 --- a/arch/sparc/lib/PeeCeeI.c +++ b/arch/sparc/lib/PeeCeeI.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * PeeCeeI.c: The emerging standard... * @@ -15,7 +16,7 @@ void outsb(unsigned long __addr, const void *src, unsigned long count) const u8 *p = src; while (count--) - outb(*p++, addr); + __raw_writeb(*p++, addr); } EXPORT_SYMBOL(outsb); @@ -93,21 +94,21 @@ void insb(unsigned long __addr, void *dst, unsigned long count) u8 *pb = dst; while ((((unsigned long)pb) & 0x3) && count--) - *pb++ = inb(addr); + *pb++ = __raw_readb(addr); pi = (u32 *)pb; while (count >= 4) { u32 w; - w = (inb(addr) << 24); - w |= (inb(addr) << 16); - w |= (inb(addr) << 8); - w |= (inb(addr) << 0); + w = (__raw_readb(addr) << 24); + w |= (__raw_readb(addr) << 16); + w |= (__raw_readb(addr) << 8); + w |= (__raw_readb(addr) << 0); *pi++ = w; count -= 4; } pb = (u8 *)pi; while (count--) - *pb++ = inb(addr); + *pb++ = __raw_readb(addr); } } EXPORT_SYMBOL(insb); @@ -121,21 +122,21 @@ void insw(unsigned long __addr, void *dst, unsigned long count) u32 *pi; if (((unsigned long)ps) & 0x2) { - *ps++ = le16_to_cpu(inw(addr)); + *ps++ = __raw_readw(addr); count--; } pi = (u32 *)ps; while (count >= 2) { u32 w; - w = (le16_to_cpu(inw(addr)) << 16); - w |= (le16_to_cpu(inw(addr)) << 0); + w = __raw_readw(addr) << 16; + w |= __raw_readw(addr) << 0; *pi++ = w; count -= 2; } ps = (u16 *)pi; if (count) - *ps = le16_to_cpu(inw(addr)); + *ps = __raw_readw(addr); } } EXPORT_SYMBOL(insw); @@ -148,7 +149,7 @@ void insl(unsigned long __addr, void *dst, unsigned long count) if ((((unsigned long)dst) & 0x3) == 0) { u32 *pi = dst; while (count--) - *pi++ = le32_to_cpu(inl(addr)); + *pi++ = __raw_readl(addr); } else { u32 l = 0, l2, *pi; u16 *ps; @@ -158,11 +159,11 @@ void insl(unsigned long __addr, void *dst, unsigned long count) case 0x2: ps = dst; count -= 1; - l = le32_to_cpu(inl(addr)); + l = __raw_readl(addr); *ps++ = l; pi = (u32 *)ps; while (count--) { - l2 = le32_to_cpu(inl(addr)); + l2 = __raw_readl(addr); *pi++ = (l << 16) | (l2 >> 16); l = l2; } @@ -173,13 +174,13 @@ void insl(unsigned long __addr, void *dst, unsigned long count) case 0x1: pb = dst; count -= 1; - l = le32_to_cpu(inl(addr)); + l = __raw_readl(addr); *pb++ = l >> 24; ps = (u16 *)pb; *ps++ = ((l >> 8) & 0xffff); pi = (u32 *)ps; while (count--) { - l2 = le32_to_cpu(inl(addr)); + l2 = __raw_readl(addr); *pi++ = (l << 24) | (l2 >> 8); l = l2; } @@ -190,11 +191,11 @@ void insl(unsigned long __addr, void *dst, unsigned long count) case 0x3: pb = (u8 *)dst; count -= 1; - l = le32_to_cpu(inl(addr)); + l = __raw_readl(addr); *pb++ = l >> 24; pi = (u32 *)pb; while (count--) { - l2 = le32_to_cpu(inl(addr)); + l2 = __raw_readl(addr); *pi++ = (l << 8) | (l2 >> 24); l = l2; } diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S index a6ae2ea04bf5..bf08d1c78836 100644 --- a/arch/sparc/lib/U1copy_from_user.S +++ b/arch/sparc/lib/U1copy_from_user.S @@ -1,17 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* U1copy_from_user.S: UltraSparc-I/II/IIi/IIe optimized copy from userspace. * * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; -#define FUNC_NAME ___copy_from_user +#define EX_LD_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define FUNC_NAME raw_copy_from_user #define LOAD(type,addr,dest) type##a [addr] %asi, dest #define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_AIUS, dest #define EX_RETVAL(x) 0 @@ -23,7 +32,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop; \ #include "U1memcpy.S" diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S index f4b970eeb485..15169851e7ab 100644 --- a/arch/sparc/lib/U1copy_to_user.S +++ b/arch/sparc/lib/U1copy_to_user.S @@ -1,17 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* U1copy_to_user.S: UltraSparc-I/II/IIi/IIe optimized copy to userspace. * * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; -#define FUNC_NAME ___copy_to_user +#define EX_ST_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define FUNC_NAME raw_copy_to_user #define STORE(type,src,addr) type##a src, [addr] ASI_AIUS #define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS #define EX_RETVAL(x) 0 @@ -23,7 +32,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop; \ #include "U1memcpy.S" diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S index b67142b7768e..154fbd35400c 100644 --- a/arch/sparc/lib/U1memcpy.S +++ b/arch/sparc/lib/U1memcpy.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* U1memcpy.S: UltraSPARC-I/II/IIi/IIe optimized memcpy. * * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com) @@ -5,6 +6,8 @@ */ #ifdef __KERNEL__ +#include <linux/export.h> +#include <linux/linkage.h> #include <asm/visasm.h> #include <asm/asi.h> #define GLOBAL_SPARE g7 @@ -23,15 +26,17 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x +#endif +#ifndef EX_LD_FP +#define EX_LD_FP(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x +#define EX_ST(x,y) x #endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#ifndef EX_ST_FP +#define EX_ST_FP(x,y) x #endif #ifndef LOAD @@ -72,53 +77,170 @@ faligndata %f7, %f8, %f60; \ faligndata %f8, %f9, %f62; -#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \ - EX_LD(LOAD_BLK(%src, %fdest)); \ - EX_ST(STORE_BLK(%fsrc, %dest)); \ - add %src, 0x40, %src; \ - subcc %len, 0x40, %len; \ - be,pn %xcc, jmptgt; \ - add %dest, 0x40, %dest; \ - -#define LOOP_CHUNK1(src, dest, len, branch_dest) \ - MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest) -#define LOOP_CHUNK2(src, dest, len, branch_dest) \ - MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest) -#define LOOP_CHUNK3(src, dest, len, branch_dest) \ - MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest) +#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt) \ + EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp); \ + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \ + add %src, 0x40, %src; \ + subcc %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE; \ + be,pn %xcc, jmptgt; \ + add %dest, 0x40, %dest; \ + +#define LOOP_CHUNK1(src, dest, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f0, f48, branch_dest) +#define LOOP_CHUNK2(src, dest, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest) +#define LOOP_CHUNK3(src, dest, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest) #define DO_SYNC membar #Sync; #define STORE_SYNC(dest, fsrc) \ - EX_ST(STORE_BLK(%fsrc, %dest)); \ + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \ add %dest, 0x40, %dest; \ DO_SYNC #define STORE_JUMP(dest, fsrc, target) \ - EX_ST(STORE_BLK(%fsrc, %dest)); \ + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp); \ add %dest, 0x40, %dest; \ ba,pt %xcc, target; \ nop; -#define FINISH_VISCHUNK(dest, f0, f1, left) \ - subcc %left, 8, %left;\ - bl,pn %xcc, 95f; \ - faligndata %f0, %f1, %f48; \ - EX_ST(STORE(std, %f48, %dest)); \ +#define FINISH_VISCHUNK(dest, f0, f1) \ + subcc %g3, 8, %g3; \ + bl,pn %xcc, 95f; \ + faligndata %f0, %f1, %f48; \ + EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp); \ add %dest, 8, %dest; -#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ - subcc %left, 8, %left; \ - bl,pn %xcc, 95f; \ +#define UNEVEN_VISCHUNK_LAST(dest, f0, f1) \ + subcc %g3, 8, %g3; \ + bl,pn %xcc, 95f; \ fsrc2 %f0, %f1; -#define UNEVEN_VISCHUNK(dest, f0, f1, left) \ - UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ +#define UNEVEN_VISCHUNK(dest, f0, f1) \ + UNEVEN_VISCHUNK_LAST(dest, f0, f1) \ ba,a,pt %xcc, 93f; .register %g2,#scratch .register %g3,#scratch .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +ENTRY(U1_g1_1_fp) + VISExitHalf + add %g1, 1, %g1 + add %g1, %g2, %g1 + retl + add %g1, %o2, %o0 +ENDPROC(U1_g1_1_fp) +ENTRY(U1_g2_0_fp) + VISExitHalf + retl + add %g2, %o2, %o0 +ENDPROC(U1_g2_0_fp) +ENTRY(U1_g2_8_fp) + VISExitHalf + add %g2, 8, %g2 + retl + add %g2, %o2, %o0 +ENDPROC(U1_g2_8_fp) +ENTRY(U1_gs_0_fp) + VISExitHalf + add %GLOBAL_SPARE, %g3, %o0 + retl + add %o0, %o2, %o0 +ENDPROC(U1_gs_0_fp) +ENTRY(U1_gs_80_fp) + VISExitHalf + add %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE + add %GLOBAL_SPARE, %g3, %o0 + retl + add %o0, %o2, %o0 +ENDPROC(U1_gs_80_fp) +ENTRY(U1_gs_40_fp) + VISExitHalf + add %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE + add %GLOBAL_SPARE, %g3, %o0 + retl + add %o0, %o2, %o0 +ENDPROC(U1_gs_40_fp) +ENTRY(U1_g3_8_fp) + VISExitHalf + add %g3, 8, %g3 + retl + add %g3, %o2, %o0 +ENDPROC(U1_g3_8_fp) +ENTRY(U1_g3_16_fp) + VISExitHalf + add %g3, 16, %g3 + retl + add %g3, %o2, %o0 +ENDPROC(U1_g3_16_fp) +ENTRY(U1_o2_0_fp) + VISExitHalf + retl + mov %o2, %o0 +ENDPROC(U1_o2_0_fp) +ENTRY(U1_o2_1_fp) + VISExitHalf + retl + add %o2, 1, %o0 +ENDPROC(U1_o2_1_fp) +ENTRY(U1_gs_0) + VISExitHalf + retl + add %GLOBAL_SPARE, %o2, %o0 +ENDPROC(U1_gs_0) +ENTRY(U1_gs_8) + VISExitHalf + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE + retl + add %GLOBAL_SPARE, 0x8, %o0 +ENDPROC(U1_gs_8) +ENTRY(U1_gs_10) + VISExitHalf + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE + retl + add %GLOBAL_SPARE, 0x10, %o0 +ENDPROC(U1_gs_10) +ENTRY(U1_o2_0) + retl + mov %o2, %o0 +ENDPROC(U1_o2_0) +ENTRY(U1_o2_8) + retl + add %o2, 8, %o0 +ENDPROC(U1_o2_8) +ENTRY(U1_o2_4) + retl + add %o2, 4, %o0 +ENDPROC(U1_o2_4) +ENTRY(U1_o2_1) + retl + add %o2, 1, %o0 +ENDPROC(U1_o2_1) +ENTRY(U1_g1_0) + retl + add %g1, %o2, %o0 +ENDPROC(U1_g1_0) +ENTRY(U1_g1_1) + add %g1, 1, %g1 + retl + add %g1, %o2, %o0 +ENDPROC(U1_g1_1) +ENTRY(U1_gs_0_o2_adj) + and %o2, 7, %o2 + retl + add %GLOBAL_SPARE, %o2, %o0 +ENDPROC(U1_gs_0_o2_adj) +ENTRY(U1_gs_8_o2_adj) + and %o2, 7, %o2 + add %GLOBAL_SPARE, 8, %GLOBAL_SPARE + retl + add %GLOBAL_SPARE, %o2, %o0 +ENDPROC(U1_gs_8_o2_adj) +#endif + .align 64 .globl FUNC_NAME @@ -160,8 +282,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ and %g2, 0x38, %g2 1: subcc %g1, 0x1, %g1 - EX_LD(LOAD(ldub, %o1 + 0x00, %o3)) - EX_ST(STORE(stb, %o3, %o1 + %GLOBAL_SPARE)) + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp) + EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp) bgu,pt %XCC, 1b add %o1, 0x1, %o1 @@ -172,20 +294,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ be,pt %icc, 3f alignaddr %o1, %g0, %o1 - EX_LD(LOAD(ldd, %o1, %f4)) -1: EX_LD(LOAD(ldd, %o1 + 0x8, %f6)) + EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp) +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f4, %f6, %f0 - EX_ST(STORE(std, %f0, %o0)) + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp) be,pn %icc, 3f add %o0, 0x8, %o0 - EX_LD(LOAD(ldd, %o1 + 0x8, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f6, %f4, %f0 - EX_ST(STORE(std, %f0, %o0)) + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp) bne,pt %icc, 1b add %o0, 0x8, %o0 @@ -208,13 +330,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ add %g1, %GLOBAL_SPARE, %g1 subcc %o2, %g3, %o2 - EX_LD(LOAD_BLK(%o1, %f0)) + EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp) add %o1, 0x40, %o1 add %g1, %g3, %g1 - EX_LD(LOAD_BLK(%o1, %f16)) + EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp) add %o1, 0x40, %o1 sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE - EX_LD(LOAD_BLK(%o1, %f32)) + EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp) add %o1, 0x40, %o1 /* There are 8 instances of the unrolled loop, @@ -234,11 +356,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 64 1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f0, %f2, %f48 1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) @@ -255,11 +377,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 56f) 1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f2, %f4, %f48 1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) @@ -276,11 +398,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 57f) 1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f4, %f6, %f48 1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) @@ -297,11 +419,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 58f) 1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f6, %f8, %f48 1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) @@ -318,11 +440,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 59f) 1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f8, %f10, %f48 1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) @@ -339,11 +461,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 60f) 1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f10, %f12, %f48 1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) @@ -360,11 +482,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 61f) 1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f12, %f14, %f48 1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) @@ -381,11 +503,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 62f) 1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f14, %f16, %f48 1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) @@ -401,53 +523,53 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) STORE_JUMP(o0, f48, 63f) -40: FINISH_VISCHUNK(o0, f0, f2, g3) -41: FINISH_VISCHUNK(o0, f2, f4, g3) -42: FINISH_VISCHUNK(o0, f4, f6, g3) -43: FINISH_VISCHUNK(o0, f6, f8, g3) -44: FINISH_VISCHUNK(o0, f8, f10, g3) -45: FINISH_VISCHUNK(o0, f10, f12, g3) -46: FINISH_VISCHUNK(o0, f12, f14, g3) -47: UNEVEN_VISCHUNK(o0, f14, f0, g3) -48: FINISH_VISCHUNK(o0, f16, f18, g3) -49: FINISH_VISCHUNK(o0, f18, f20, g3) -50: FINISH_VISCHUNK(o0, f20, f22, g3) -51: FINISH_VISCHUNK(o0, f22, f24, g3) -52: FINISH_VISCHUNK(o0, f24, f26, g3) -53: FINISH_VISCHUNK(o0, f26, f28, g3) -54: FINISH_VISCHUNK(o0, f28, f30, g3) -55: UNEVEN_VISCHUNK(o0, f30, f0, g3) -56: FINISH_VISCHUNK(o0, f32, f34, g3) -57: FINISH_VISCHUNK(o0, f34, f36, g3) -58: FINISH_VISCHUNK(o0, f36, f38, g3) -59: FINISH_VISCHUNK(o0, f38, f40, g3) -60: FINISH_VISCHUNK(o0, f40, f42, g3) -61: FINISH_VISCHUNK(o0, f42, f44, g3) -62: FINISH_VISCHUNK(o0, f44, f46, g3) -63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3) - -93: EX_LD(LOAD(ldd, %o1, %f2)) +40: FINISH_VISCHUNK(o0, f0, f2) +41: FINISH_VISCHUNK(o0, f2, f4) +42: FINISH_VISCHUNK(o0, f4, f6) +43: FINISH_VISCHUNK(o0, f6, f8) +44: FINISH_VISCHUNK(o0, f8, f10) +45: FINISH_VISCHUNK(o0, f10, f12) +46: FINISH_VISCHUNK(o0, f12, f14) +47: UNEVEN_VISCHUNK(o0, f14, f0) +48: FINISH_VISCHUNK(o0, f16, f18) +49: FINISH_VISCHUNK(o0, f18, f20) +50: FINISH_VISCHUNK(o0, f20, f22) +51: FINISH_VISCHUNK(o0, f22, f24) +52: FINISH_VISCHUNK(o0, f24, f26) +53: FINISH_VISCHUNK(o0, f26, f28) +54: FINISH_VISCHUNK(o0, f28, f30) +55: UNEVEN_VISCHUNK(o0, f30, f0) +56: FINISH_VISCHUNK(o0, f32, f34) +57: FINISH_VISCHUNK(o0, f34, f36) +58: FINISH_VISCHUNK(o0, f36, f38) +59: FINISH_VISCHUNK(o0, f38, f40) +60: FINISH_VISCHUNK(o0, f40, f42) +61: FINISH_VISCHUNK(o0, f42, f44) +62: FINISH_VISCHUNK(o0, f44, f46) +63: UNEVEN_VISCHUNK_LAST(o0, f46, f0) + +93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_8_fp) add %o1, 8, %o1 subcc %g3, 8, %g3 faligndata %f0, %f2, %f8 - EX_ST(STORE(std, %f8, %o0)) + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_16_fp) bl,pn %xcc, 95f add %o0, 8, %o0 - EX_LD(LOAD(ldd, %o1, %f0)) + EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_8_fp) add %o1, 8, %o1 subcc %g3, 8, %g3 faligndata %f2, %f0, %f8 - EX_ST(STORE(std, %f8, %o0)) + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_16_fp) bge,pt %xcc, 93b add %o0, 8, %o0 95: brz,pt %o2, 2f mov %g1, %o1 -1: EX_LD(LOAD(ldub, %o1, %o3)) +1: EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp) add %o1, 1, %o1 subcc %o2, 1, %o2 - EX_ST(STORE(stb, %o3, %o0)) + EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp) bne,pt %xcc, 1b add %o0, 1, %o0 @@ -463,27 +585,27 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 72: andn %o2, 0xf, %GLOBAL_SPARE and %o2, 0xf, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5)) - EX_LD(LOAD(ldx, %o1 + 0x08, %g1)) +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0) + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0) subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10) add %o1, 0x8, %o1 - EX_ST(STORE(stx, %g1, %o1 + %o3)) + EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8) bgu,pt %XCC, 1b add %o1, 0x8, %o1 73: andcc %o2, 0x8, %g0 be,pt %XCC, 1f nop - EX_LD(LOAD(ldx, %o1, %o5)) + EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0) sub %o2, 0x8, %o2 - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8) add %o1, 0x8, %o1 1: andcc %o2, 0x4, %g0 be,pt %XCC, 1f nop - EX_LD(LOAD(lduw, %o1, %o5)) + EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0) sub %o2, 0x4, %o2 - EX_ST(STORE(stw, %o5, %o1 + %o3)) + EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4) add %o1, 0x4, %o1 1: cmp %o2, 0 be,pt %XCC, 85f @@ -497,9 +619,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %g0, %g1, %g1 sub %o2, %g1, %o2 -1: EX_LD(LOAD(ldub, %o1, %o5)) +1: EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0) subcc %g1, 1, %g1 - EX_ST(STORE(stb, %o5, %o1 + %o3)) + EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1) bgu,pt %icc, 1b add %o1, 1, %o1 @@ -515,16 +637,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 8: mov 64, %o3 andn %o1, 0x7, %o1 - EX_LD(LOAD(ldx, %o1, %g2)) + EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0) sub %o3, %g1, %o3 andn %o2, 0x7, %GLOBAL_SPARE sllx %g2, %g1, %g2 -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3)) +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj) subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE add %o1, 0x8, %o1 srlx %g3, %o3, %o5 or %o5, %g2, %o5 - EX_ST(STORE(stx, %o5, %o0)) + EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj) add %o0, 0x8, %o0 bgu,pt %icc, 1b sllx %g3, %g1, %g2 @@ -542,9 +664,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ bne,pn %XCC, 90f sub %o0, %o1, %o3 -1: EX_LD(LOAD(lduw, %o1, %g1)) +1: EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0) subcc %o2, 4, %o2 - EX_ST(STORE(stw, %g1, %o1 + %o3)) + EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4) bgu,pt %XCC, 1b add %o1, 4, %o1 @@ -552,12 +674,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ mov EX_RETVAL(%o4), %o0 .align 32 -90: EX_LD(LOAD(ldub, %o1, %g1)) +90: EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0) subcc %o2, 1, %o2 - EX_ST(STORE(stb, %g1, %o1 + %o3)) + EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1) bgu,pt %XCC, 90b add %o1, 1, %o1 retl mov EX_RETVAL(%o4), %o0 .size FUNC_NAME, .-FUNC_NAME +EXPORT_SYMBOL(FUNC_NAME) diff --git a/arch/sparc/lib/U3copy_from_user.S b/arch/sparc/lib/U3copy_from_user.S index b1acd1331c33..9c891e9edc7b 100644 --- a/arch/sparc/lib/U3copy_from_user.S +++ b/arch/sparc/lib/U3copy_from_user.S @@ -1,13 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* U3copy_from_user.S: UltraSparc-III optimized copy from userspace. * * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_LD_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y##_fp; \ .text; \ .align 4; diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S index ef1e493afdfa..da424608272c 100644 --- a/arch/sparc/lib/U3copy_to_user.S +++ b/arch/sparc/lib/U3copy_to_user.S @@ -1,13 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* U3copy_to_user.S: UltraSparc-III optimized copy to userspace. * * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_ST_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y##_fp; \ .text; \ .align 4; @@ -23,7 +32,7 @@ #define PREAMBLE \ rd %asi, %g1; \ cmp %g1, ASI_AIUS; \ - bne,pn %icc, ___copy_in_user; \ + bne,pn %icc, raw_copy_in_user; \ nop; \ #include "U3memcpy.S" diff --git a/arch/sparc/lib/U3memcpy.S b/arch/sparc/lib/U3memcpy.S index 7cae9cc6a204..bace3a18f836 100644 --- a/arch/sparc/lib/U3memcpy.S +++ b/arch/sparc/lib/U3memcpy.S @@ -1,9 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* U3memcpy.S: UltraSparc-III optimized memcpy. * * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ #ifdef __KERNEL__ +#include <linux/linkage.h> #include <asm/visasm.h> #include <asm/asi.h> #define GLOBAL_SPARE %g7 @@ -22,15 +24,17 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x +#endif +#ifndef EX_LD_FP +#define EX_LD_FP(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x +#define EX_ST(x,y) x #endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#ifndef EX_ST_FP +#define EX_ST_FP(x,y) x #endif #ifndef LOAD @@ -71,6 +75,87 @@ */ .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_fp: + VISExitHalf + retl + nop +ENTRY(U3_retl_o2_plus_g2_plus_g1_plus_1_fp) + add %g1, 1, %g1 + add %g2, %g1, %g2 + ba,pt %xcc, __restore_fp + add %o2, %g2, %o0 +ENDPROC(U3_retl_o2_plus_g2_plus_g1_plus_1_fp) +ENTRY(U3_retl_o2_plus_g2_fp) + ba,pt %xcc, __restore_fp + add %o2, %g2, %o0 +ENDPROC(U3_retl_o2_plus_g2_fp) +ENTRY(U3_retl_o2_plus_g2_plus_8_fp) + add %g2, 8, %g2 + ba,pt %xcc, __restore_fp + add %o2, %g2, %o0 +ENDPROC(U3_retl_o2_plus_g2_plus_8_fp) +ENTRY(U3_retl_o2) + retl + mov %o2, %o0 +ENDPROC(U3_retl_o2) +ENTRY(U3_retl_o2_plus_1) + retl + add %o2, 1, %o0 +ENDPROC(U3_retl_o2_plus_1) +ENTRY(U3_retl_o2_plus_4) + retl + add %o2, 4, %o0 +ENDPROC(U3_retl_o2_plus_4) +ENTRY(U3_retl_o2_plus_8) + retl + add %o2, 8, %o0 +ENDPROC(U3_retl_o2_plus_8) +ENTRY(U3_retl_o2_plus_g1_plus_1) + add %g1, 1, %g1 + retl + add %o2, %g1, %o0 +ENDPROC(U3_retl_o2_plus_g1_plus_1) +ENTRY(U3_retl_o2_fp) + ba,pt %xcc, __restore_fp + mov %o2, %o0 +ENDPROC(U3_retl_o2_fp) +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp) + sll %o3, 6, %o3 + add %o3, 0x80, %o3 + ba,pt %xcc, __restore_fp + add %o2, %o3, %o0 +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp) +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp) + sll %o3, 6, %o3 + add %o3, 0x40, %o3 + ba,pt %xcc, __restore_fp + add %o2, %o3, %o0 +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp) +ENTRY(U3_retl_o2_plus_GS_plus_0x10) + add GLOBAL_SPARE, 0x10, GLOBAL_SPARE + retl + add %o2, GLOBAL_SPARE, %o0 +ENDPROC(U3_retl_o2_plus_GS_plus_0x10) +ENTRY(U3_retl_o2_plus_GS_plus_0x08) + add GLOBAL_SPARE, 0x08, GLOBAL_SPARE + retl + add %o2, GLOBAL_SPARE, %o0 +ENDPROC(U3_retl_o2_plus_GS_plus_0x08) +ENTRY(U3_retl_o2_and_7_plus_GS) + and %o2, 7, %o2 + retl + add %o2, GLOBAL_SPARE, %o0 +ENDPROC(U3_retl_o2_and_7_plus_GS) +ENTRY(U3_retl_o2_and_7_plus_GS_plus_8) + add GLOBAL_SPARE, 8, GLOBAL_SPARE + and %o2, 7, %o2 + retl + add %o2, GLOBAL_SPARE, %o0 +ENDPROC(U3_retl_o2_and_7_plus_GS_plus_8) +#endif + .align 64 /* The cheetah's flexible spine, oversized liver, enlarged heart, @@ -84,18 +169,25 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ srlx %o2, 31, %g2 cmp %g2, 0 + + /* software trap 5 "Range Check" if dst >= 0x80000000 */ tne %xcc, 5 PREAMBLE mov %o0, %o4 + + /* if len == 0 */ cmp %o2, 0 - be,pn %XCC, 85f + be,pn %XCC, end_return or %o0, %o1, %o3 + + /* if len < 16 */ cmp %o2, 16 - blu,a,pn %XCC, 80f + blu,a,pn %XCC, less_than_16 or %o3, %o2, %o3 + /* if len < 192 */ cmp %o2, (3 * 64) - blu,pt %XCC, 70f + blu,pt %XCC, less_than_192 andcc %o3, 0x7, %g0 /* Clobbers o5/g1/g2/g3/g7/icc/xcc. We must preserve @@ -120,8 +212,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ and %g2, 0x38, %g2 1: subcc %g1, 0x1, %g1 - EX_LD(LOAD(ldub, %o1 + 0x00, %o3)) - EX_ST(STORE(stb, %o3, %o1 + GLOBAL_SPARE)) + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U3_retl_o2_plus_g2_plus_g1_plus_1) + EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE), U3_retl_o2_plus_g2_plus_g1_plus_1) bgu,pt %XCC, 1b add %o1, 0x1, %o1 @@ -132,20 +224,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ be,pt %icc, 3f alignaddr %o1, %g0, %o1 - EX_LD(LOAD(ldd, %o1, %f4)) -1: EX_LD(LOAD(ldd, %o1 + 0x8, %f6)) + EX_LD_FP(LOAD(ldd, %o1, %f4), U3_retl_o2_plus_g2) +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U3_retl_o2_plus_g2) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f4, %f6, %f0 - EX_ST(STORE(std, %f0, %o0)) + EX_ST_FP(STORE(std, %f0, %o0), U3_retl_o2_plus_g2_plus_8) be,pn %icc, 3f add %o0, 0x8, %o0 - EX_LD(LOAD(ldd, %o1 + 0x8, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U3_retl_o2_plus_g2) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f6, %f4, %f2 - EX_ST(STORE(std, %f2, %o0)) + EX_ST_FP(STORE(std, %f2, %o0), U3_retl_o2_plus_g2_plus_8) bne,pt %icc, 1b add %o0, 0x8, %o0 @@ -155,26 +247,27 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ LOAD(prefetch, %o1 + 0x080, #one_read) LOAD(prefetch, %o1 + 0x0c0, #one_read) LOAD(prefetch, %o1 + 0x100, #one_read) - EX_LD(LOAD(ldd, %o1 + 0x000, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0), U3_retl_o2) LOAD(prefetch, %o1 + 0x140, #one_read) - EX_LD(LOAD(ldd, %o1 + 0x008, %f2)) + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2) LOAD(prefetch, %o1 + 0x180, #one_read) - EX_LD(LOAD(ldd, %o1 + 0x010, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2) LOAD(prefetch, %o1 + 0x1c0, #one_read) faligndata %f0, %f2, %f16 - EX_LD(LOAD(ldd, %o1 + 0x018, %f6)) + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2) faligndata %f2, %f4, %f18 - EX_LD(LOAD(ldd, %o1 + 0x020, %f8)) + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2) faligndata %f4, %f6, %f20 - EX_LD(LOAD(ldd, %o1 + 0x028, %f10)) + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2) faligndata %f6, %f8, %f22 - EX_LD(LOAD(ldd, %o1 + 0x030, %f12)) + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2) faligndata %f8, %f10, %f24 - EX_LD(LOAD(ldd, %o1 + 0x038, %f14)) + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2) faligndata %f10, %f12, %f26 - EX_LD(LOAD(ldd, %o1 + 0x040, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2) + and %o2, 0x3f, %o2 subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE add %o1, 0x40, %o1 bgu,pt %XCC, 1f @@ -184,26 +277,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 64 1: - EX_LD(LOAD(ldd, %o1 + 0x008, %f2)) + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f12, %f14, %f28 - EX_LD(LOAD(ldd, %o1 + 0x010, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f14, %f0, %f30 - EX_ST(STORE_BLK(%f16, %o0)) - EX_LD(LOAD(ldd, %o1 + 0x018, %f6)) + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80) + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f0, %f2, %f16 add %o0, 0x40, %o0 - EX_LD(LOAD(ldd, %o1 + 0x020, %f8)) + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f2, %f4, %f18 - EX_LD(LOAD(ldd, %o1 + 0x028, %f10)) + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f4, %f6, %f20 - EX_LD(LOAD(ldd, %o1 + 0x030, %f12)) + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40) subcc %o3, 0x01, %o3 faligndata %f6, %f8, %f22 - EX_LD(LOAD(ldd, %o1 + 0x038, %f14)) + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f8, %f10, %f24 - EX_LD(LOAD(ldd, %o1 + 0x040, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x80) LOAD(prefetch, %o1 + 0x1c0, #one_read) faligndata %f10, %f12, %f26 bg,pt %XCC, 1b @@ -211,29 +304,29 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ /* Finally we copy the last full 64-byte block. */ 2: - EX_LD(LOAD(ldd, %o1 + 0x008, %f2)) + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f12, %f14, %f28 - EX_LD(LOAD(ldd, %o1 + 0x010, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f14, %f0, %f30 - EX_ST(STORE_BLK(%f16, %o0)) - EX_LD(LOAD(ldd, %o1 + 0x018, %f6)) + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80) + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f0, %f2, %f16 - EX_LD(LOAD(ldd, %o1 + 0x020, %f8)) + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f2, %f4, %f18 - EX_LD(LOAD(ldd, %o1 + 0x028, %f10)) + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f4, %f6, %f20 - EX_LD(LOAD(ldd, %o1 + 0x030, %f12)) + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f6, %f8, %f22 - EX_LD(LOAD(ldd, %o1 + 0x038, %f14)) + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f8, %f10, %f24 cmp %g1, 0 be,pt %XCC, 1f add %o0, 0x40, %o0 - EX_LD(LOAD(ldd, %o1 + 0x040, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x40) 1: faligndata %f10, %f12, %f26 faligndata %f12, %f14, %f28 faligndata %f14, %f0, %f30 - EX_ST(STORE_BLK(%f16, %o0)) + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x40) add %o0, 0x40, %o0 add %o1, 0x40, %o1 membar #Sync @@ -244,7 +337,6 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ * Also notice how this code is careful not to perform a * load past the end of the src buffer. */ - and %o2, 0x3f, %o2 andcc %o2, 0x38, %g2 be,pn %XCC, 2f subcc %g2, 0x8, %g2 @@ -253,20 +345,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %o2, %g2, %o2 be,a,pt %XCC, 1f - EX_LD(LOAD(ldd, %o1 + 0x00, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0), U3_retl_o2_plus_g2) -1: EX_LD(LOAD(ldd, %o1 + 0x08, %f2)) +1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2), U3_retl_o2_plus_g2) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f0, %f2, %f8 - EX_ST(STORE(std, %f8, %o0)) + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8) be,pn %XCC, 2f add %o0, 0x8, %o0 - EX_LD(LOAD(ldd, %o1 + 0x08, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0), U3_retl_o2_plus_g2) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f2, %f0, %f8 - EX_ST(STORE(std, %f8, %o0)) + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8) bne,pn %XCC, 1b add %o0, 0x8, %o0 @@ -278,7 +370,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ cmp %o2, 0 add %o1, %g1, %o1 VISExitHalf - be,pn %XCC, 85f + be,pn %XCC, end_return sub %o0, %o1, %o3 andcc %g1, 0x7, %g0 @@ -286,33 +378,37 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andcc %o2, 0x8, %g0 be,pt %icc, 1f nop - EX_LD(LOAD(ldx, %o1, %o5)) - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2) + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2) add %o1, 0x8, %o1 + sub %o2, 8, %o2 1: andcc %o2, 0x4, %g0 be,pt %icc, 1f nop - EX_LD(LOAD(lduw, %o1, %o5)) - EX_ST(STORE(stw, %o5, %o1 + %o3)) + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2) + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2) add %o1, 0x4, %o1 + sub %o2, 4, %o2 1: andcc %o2, 0x2, %g0 be,pt %icc, 1f nop - EX_LD(LOAD(lduh, %o1, %o5)) - EX_ST(STORE(sth, %o5, %o1 + %o3)) + EX_LD(LOAD(lduh, %o1, %o5), U3_retl_o2) + EX_ST(STORE(sth, %o5, %o1 + %o3), U3_retl_o2) add %o1, 0x2, %o1 + sub %o2, 2, %o2 1: andcc %o2, 0x1, %g0 - be,pt %icc, 85f + be,pt %icc, end_return nop - EX_LD(LOAD(ldub, %o1, %o5)) - ba,pt %xcc, 85f - EX_ST(STORE(stb, %o5, %o1 + %o3)) + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2) + ba,pt %xcc, end_return + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2) .align 64 -70: /* 16 < len <= 64 */ + /* 16 <= len < 192 */ +less_than_192: bne,pn %XCC, 75f sub %o0, %o1, %o3 @@ -320,29 +416,29 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0xf, GLOBAL_SPARE and %o2, 0xf, %o2 1: subcc GLOBAL_SPARE, 0x10, GLOBAL_SPARE - EX_LD(LOAD(ldx, %o1 + 0x00, %o5)) - EX_LD(LOAD(ldx, %o1 + 0x08, %g1)) - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U3_retl_o2_plus_GS_plus_0x10) + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U3_retl_o2_plus_GS_plus_0x10) + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x10) add %o1, 0x8, %o1 - EX_ST(STORE(stx, %g1, %o1 + %o3)) + EX_ST(STORE(stx, %g1, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x08) bgu,pt %XCC, 1b add %o1, 0x8, %o1 73: andcc %o2, 0x8, %g0 be,pt %XCC, 1f nop sub %o2, 0x8, %o2 - EX_LD(LOAD(ldx, %o1, %o5)) - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2_plus_8) + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_8) add %o1, 0x8, %o1 1: andcc %o2, 0x4, %g0 be,pt %XCC, 1f nop sub %o2, 0x4, %o2 - EX_LD(LOAD(lduw, %o1, %o5)) - EX_ST(STORE(stw, %o5, %o1 + %o3)) + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2_plus_4) + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2_plus_4) add %o1, 0x4, %o1 1: cmp %o2, 0 - be,pt %XCC, 85f + be,pt %XCC, end_return nop ba,pt %xcc, 90f nop @@ -355,8 +451,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %o2, %g1, %o2 1: subcc %g1, 1, %g1 - EX_LD(LOAD(ldub, %o1, %o5)) - EX_ST(STORE(stb, %o5, %o1 + %o3)) + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2_plus_g1_plus_1) + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2_plus_g1_plus_1) bgu,pt %icc, 1b add %o1, 1, %o1 @@ -372,48 +468,50 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 8: mov 64, %o3 andn %o1, 0x7, %o1 - EX_LD(LOAD(ldx, %o1, %g2)) + EX_LD(LOAD(ldx, %o1, %g2), U3_retl_o2) sub %o3, %g1, %o3 andn %o2, 0x7, GLOBAL_SPARE sllx %g2, %g1, %g2 -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3)) +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U3_retl_o2_and_7_plus_GS) subcc GLOBAL_SPARE, 0x8, GLOBAL_SPARE add %o1, 0x8, %o1 srlx %g3, %o3, %o5 or %o5, %g2, %o5 - EX_ST(STORE(stx, %o5, %o0)) + EX_ST(STORE(stx, %o5, %o0), U3_retl_o2_and_7_plus_GS_plus_8) add %o0, 0x8, %o0 bgu,pt %icc, 1b sllx %g3, %g1, %g2 srl %g1, 3, %g1 andcc %o2, 0x7, %o2 - be,pn %icc, 85f + be,pn %icc, end_return add %o1, %g1, %o1 ba,pt %xcc, 90f sub %o0, %o1, %o3 .align 64 -80: /* 0 < len <= 16 */ + /* 0 < len < 16 */ +less_than_16: andcc %o3, 0x3, %g0 bne,pn %XCC, 90f sub %o0, %o1, %o3 1: subcc %o2, 4, %o2 - EX_LD(LOAD(lduw, %o1, %g1)) - EX_ST(STORE(stw, %g1, %o1 + %o3)) + EX_LD(LOAD(lduw, %o1, %g1), U3_retl_o2_plus_4) + EX_ST(STORE(stw, %g1, %o1 + %o3), U3_retl_o2_plus_4) bgu,pt %XCC, 1b add %o1, 4, %o1 -85: retl +end_return: + retl mov EX_RETVAL(%o4), %o0 .align 32 90: subcc %o2, 1, %o2 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o1 + %o3)) + EX_LD(LOAD(ldub, %o1, %g1), U3_retl_o2_plus_1) + EX_ST(STORE(stb, %g1, %o1 + %o3), U3_retl_o2_plus_1) bgu,pt %XCC, 90b add %o1, 1, %o1 retl diff --git a/arch/sparc/lib/U3patch.S b/arch/sparc/lib/U3patch.S index ecc302619a6e..9a888088f3c9 100644 --- a/arch/sparc/lib/U3patch.S +++ b/arch/sparc/lib/U3patch.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* U3patch.S: Patch Ultra-I routines with Ultra-III variant. * * Copyright (C) 2004 David S. Miller <davem@redhat.com> @@ -26,8 +27,8 @@ .type cheetah_patch_copyops,#function cheetah_patch_copyops: ULTRA3_DO_PATCH(memcpy, U3memcpy) - ULTRA3_DO_PATCH(___copy_from_user, U3copy_from_user) - ULTRA3_DO_PATCH(___copy_to_user, U3copy_to_user) + ULTRA3_DO_PATCH(raw_copy_from_user, U3copy_from_user) + ULTRA3_DO_PATCH(raw_copy_to_user, U3copy_to_user) retl nop .size cheetah_patch_copyops,.-cheetah_patch_copyops diff --git a/arch/sparc/lib/VISsave.S b/arch/sparc/lib/VISsave.S index b320ae9e2e2e..31a0c336c185 100644 --- a/arch/sparc/lib/VISsave.S +++ b/arch/sparc/lib/VISsave.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * VISsave.S: Code for saving FPU register state for * VIS routines. One should not call this directly, @@ -6,24 +7,24 @@ * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) */ +#include <linux/export.h> +#include <linux/linkage.h> + #include <asm/asi.h> #include <asm/page.h> #include <asm/ptrace.h> #include <asm/visasm.h> #include <asm/thread_info.h> - .text - .globl VISenter, VISenterhalf - /* On entry: %o5=current FPRS value, %g7 is callers address */ /* May clobber %o5, %g1, %g2, %g3, %g7, %icc, %xcc */ /* Nothing special need be done here to handle pre-emption, this * FPU save/restore mechanism is already preemption safe. */ - + .text .align 32 -VISenter: +ENTRY(VISenter) ldub [%g6 + TI_FPDEPTH], %g1 brnz,a,pn %g1, 1f cmp %g1, 1 @@ -44,9 +45,8 @@ vis1: ldub [%g6 + TI_FPSAVED], %g3 stx %g3, [%g6 + TI_GSR] 2: add %g6, %g1, %g3 - cmp %o5, FPRS_DU - be,pn %icc, 6f - sll %g1, 3, %g1 + mov FPRS_DU | FPRS_DL | FPRS_FEF, %o5 + sll %g1, 3, %g1 stb %o5, [%g3 + TI_FPSAVED] rd %gsr, %g2 add %g6, %g1, %g3 @@ -80,65 +80,5 @@ vis1: ldub [%g6 + TI_FPSAVED], %g3 .align 32 80: jmpl %g7 + %g0, %g0 nop - -6: ldub [%g3 + TI_FPSAVED], %o5 - or %o5, FPRS_DU, %o5 - add %g6, TI_FPREGS+0x80, %g2 - stb %o5, [%g3 + TI_FPSAVED] - - sll %g1, 5, %g1 - add %g6, TI_FPREGS+0xc0, %g3 - wr %g0, FPRS_FEF, %fprs - membar #Sync - stda %f32, [%g2 + %g1] ASI_BLK_P - stda %f48, [%g3 + %g1] ASI_BLK_P - membar #Sync - ba,pt %xcc, 80f - nop - - .align 32 -80: jmpl %g7 + %g0, %g0 - nop - - .align 32 -VISenterhalf: - ldub [%g6 + TI_FPDEPTH], %g1 - brnz,a,pn %g1, 1f - cmp %g1, 1 - stb %g0, [%g6 + TI_FPSAVED] - stx %fsr, [%g6 + TI_XFSR] - clr %o5 - jmpl %g7 + %g0, %g0 - wr %g0, FPRS_FEF, %fprs - -1: bne,pn %icc, 2f - srl %g1, 1, %g1 - ba,pt %xcc, vis1 - sub %g7, 8, %g7 -2: addcc %g6, %g1, %g3 - sll %g1, 3, %g1 - andn %o5, FPRS_DU, %g2 - stb %g2, [%g3 + TI_FPSAVED] - - rd %gsr, %g2 - add %g6, %g1, %g3 - stx %g2, [%g3 + TI_GSR] - add %g6, %g1, %g2 - stx %fsr, [%g2 + TI_XFSR] - sll %g1, 5, %g1 -3: andcc %o5, FPRS_DL, %g0 - be,pn %icc, 4f - add %g6, TI_FPREGS, %g2 - - add %g6, TI_FPREGS+0x40, %g3 - membar #Sync - stda %f0, [%g2 + %g1] ASI_BLK_P - stda %f16, [%g3 + %g1] ASI_BLK_P - membar #Sync - ba,pt %xcc, 4f - nop - - .align 32 -4: and %o5, FPRS_DU, %o5 - jmpl %g7 + %g0, %g0 - wr %o5, FPRS_FEF, %fprs +ENDPROC(VISenter) +EXPORT_SYMBOL(VISenter) diff --git a/arch/sparc/lib/ashldi3.S b/arch/sparc/lib/ashldi3.S index 86f60de07b0a..2a9e7c4fb260 100644 --- a/arch/sparc/lib/ashldi3.S +++ b/arch/sparc/lib/ashldi3.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * ashldi3.S: GCC emits these for certain drivers playing * with long longs. @@ -5,6 +6,7 @@ * Copyright (C) 1999 David S. Miller (davem@redhat.com) */ +#include <linux/export.h> #include <linux/linkage.h> .text @@ -33,3 +35,4 @@ ENTRY(__ashldi3) retl nop ENDPROC(__ashldi3) +EXPORT_SYMBOL(__ashldi3) diff --git a/arch/sparc/lib/ashrdi3.S b/arch/sparc/lib/ashrdi3.S index 6eb8ba2dd50e..8fd0b311722f 100644 --- a/arch/sparc/lib/ashrdi3.S +++ b/arch/sparc/lib/ashrdi3.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * ashrdi3.S: The filesystem code creates all kinds of references to * this little routine on the sparc with gcc. @@ -5,6 +6,7 @@ * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) */ +#include <linux/export.h> #include <linux/linkage.h> .text @@ -35,3 +37,4 @@ ENTRY(__ashrdi3) jmpl %o7 + 8, %g0 nop ENDPROC(__ashrdi3) +EXPORT_SYMBOL(__ashrdi3) diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c index 1d32b54089aa..8ae880ebf07a 100644 --- a/arch/sparc/lib/atomic32.c +++ b/arch/sparc/lib/atomic32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * atomic32.c: 32-bit atomic_t implementation * @@ -27,20 +28,59 @@ static DEFINE_SPINLOCK(dummy); #endif /* SMP */ -int __atomic_add_return(int i, atomic_t *v) +#define ATOMIC_FETCH_OP(op, c_op) \ +int arch_atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + int ret; \ + unsigned long flags; \ + spin_lock_irqsave(ATOMIC_HASH(v), flags); \ + \ + ret = v->counter; \ + v->counter c_op i; \ + \ + spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ + return ret; \ +} \ +EXPORT_SYMBOL(arch_atomic_fetch_##op); + +#define ATOMIC_OP_RETURN(op, c_op) \ +int arch_atomic_##op##_return(int i, atomic_t *v) \ +{ \ + int ret; \ + unsigned long flags; \ + spin_lock_irqsave(ATOMIC_HASH(v), flags); \ + \ + ret = (v->counter c_op i); \ + \ + spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ + return ret; \ +} \ +EXPORT_SYMBOL(arch_atomic_##op##_return); + +ATOMIC_OP_RETURN(add, +=) + +ATOMIC_FETCH_OP(add, +=) +ATOMIC_FETCH_OP(and, &=) +ATOMIC_FETCH_OP(or, |=) +ATOMIC_FETCH_OP(xor, ^=) + +#undef ATOMIC_FETCH_OP +#undef ATOMIC_OP_RETURN + +int arch_atomic_xchg(atomic_t *v, int new) { int ret; unsigned long flags; - spin_lock_irqsave(ATOMIC_HASH(v), flags); - - ret = (v->counter += i); + spin_lock_irqsave(ATOMIC_HASH(v), flags); + ret = v->counter; + v->counter = new; spin_unlock_irqrestore(ATOMIC_HASH(v), flags); return ret; } -EXPORT_SYMBOL(__atomic_add_return); +EXPORT_SYMBOL(arch_atomic_xchg); -int atomic_cmpxchg(atomic_t *v, int old, int new) +int arch_atomic_cmpxchg(atomic_t *v, int old, int new) { int ret; unsigned long flags; @@ -53,9 +93,9 @@ int atomic_cmpxchg(atomic_t *v, int old, int new) spin_unlock_irqrestore(ATOMIC_HASH(v), flags); return ret; } -EXPORT_SYMBOL(atomic_cmpxchg); +EXPORT_SYMBOL(arch_atomic_cmpxchg); -int __atomic_add_unless(atomic_t *v, int a, int u) +int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) { int ret; unsigned long flags; @@ -67,10 +107,10 @@ int __atomic_add_unless(atomic_t *v, int a, int u) spin_unlock_irqrestore(ATOMIC_HASH(v), flags); return ret; } -EXPORT_SYMBOL(__atomic_add_unless); +EXPORT_SYMBOL(arch_atomic_fetch_add_unless); /* Atomic operations are already serializing */ -void atomic_set(atomic_t *v, int i) +void arch_atomic_set(atomic_t *v, int i) { unsigned long flags; @@ -78,9 +118,9 @@ void atomic_set(atomic_t *v, int i) v->counter = i; spin_unlock_irqrestore(ATOMIC_HASH(v), flags); } -EXPORT_SYMBOL(atomic_set); +EXPORT_SYMBOL(arch_atomic_set); -unsigned long ___set_bit(unsigned long *addr, unsigned long mask) +unsigned long sp32___set_bit(unsigned long *addr, unsigned long mask) { unsigned long old, flags; @@ -91,9 +131,9 @@ unsigned long ___set_bit(unsigned long *addr, unsigned long mask) return old & mask; } -EXPORT_SYMBOL(___set_bit); +EXPORT_SYMBOL(sp32___set_bit); -unsigned long ___clear_bit(unsigned long *addr, unsigned long mask) +unsigned long sp32___clear_bit(unsigned long *addr, unsigned long mask) { unsigned long old, flags; @@ -104,9 +144,9 @@ unsigned long ___clear_bit(unsigned long *addr, unsigned long mask) return old & mask; } -EXPORT_SYMBOL(___clear_bit); +EXPORT_SYMBOL(sp32___clear_bit); -unsigned long ___change_bit(unsigned long *addr, unsigned long mask) +unsigned long sp32___change_bit(unsigned long *addr, unsigned long mask) { unsigned long old, flags; @@ -117,18 +157,41 @@ unsigned long ___change_bit(unsigned long *addr, unsigned long mask) return old & mask; } -EXPORT_SYMBOL(___change_bit); +EXPORT_SYMBOL(sp32___change_bit); + +#define CMPXCHG(T) \ + T __cmpxchg_##T(volatile T *ptr, T old, T new) \ + { \ + unsigned long flags; \ + T prev; \ + \ + spin_lock_irqsave(ATOMIC_HASH(ptr), flags); \ + if ((prev = *ptr) == old) \ + *ptr = new; \ + spin_unlock_irqrestore(ATOMIC_HASH(ptr), flags);\ + \ + return prev; \ + } + +CMPXCHG(u8) +CMPXCHG(u16) +CMPXCHG(u32) +CMPXCHG(u64) +EXPORT_SYMBOL(__cmpxchg_u8); +EXPORT_SYMBOL(__cmpxchg_u16); +EXPORT_SYMBOL(__cmpxchg_u32); +EXPORT_SYMBOL(__cmpxchg_u64); -unsigned long __cmpxchg_u32(volatile u32 *ptr, u32 old, u32 new) +unsigned long __xchg_u32(volatile u32 *ptr, u32 new) { unsigned long flags; u32 prev; spin_lock_irqsave(ATOMIC_HASH(ptr), flags); - if ((prev = *ptr) == old) - *ptr = new; + prev = *ptr; + *ptr = new; spin_unlock_irqrestore(ATOMIC_HASH(ptr), flags); return (unsigned long)prev; } -EXPORT_SYMBOL(__cmpxchg_u32); +EXPORT_SYMBOL(__xchg_u32); diff --git a/arch/sparc/lib/atomic_64.S b/arch/sparc/lib/atomic_64.S index 85c233d0a340..4f8cab2fb9cd 100644 --- a/arch/sparc/lib/atomic_64.S +++ b/arch/sparc/lib/atomic_64.S @@ -1,124 +1,156 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* atomic.S: These things are too big to do inline. * * Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net) */ +#include <linux/export.h> #include <linux/linkage.h> #include <asm/asi.h> #include <asm/backoff.h> .text - /* Two versions of the atomic routines, one that + /* Three versions of the atomic routines, one that * does not return a value and does not perform - * memory barriers, and a second which returns - * a value and does the barriers. + * memory barriers, and a two which return + * a value, the new and old value resp. and does the + * barriers. */ -ENTRY(atomic_add) /* %o0 = increment, %o1 = atomic_ptr */ - BACKOFF_SETUP(%o2) -1: lduw [%o1], %g1 - add %g1, %o0, %g7 - cas [%o1], %g1, %g7 - cmp %g1, %g7 - bne,pn %icc, BACKOFF_LABEL(2f, 1b) - nop - retl - nop -2: BACKOFF_SPIN(%o2, %o3, 1b) -ENDPROC(atomic_add) -ENTRY(atomic_sub) /* %o0 = decrement, %o1 = atomic_ptr */ - BACKOFF_SETUP(%o2) -1: lduw [%o1], %g1 - sub %g1, %o0, %g7 - cas [%o1], %g1, %g7 - cmp %g1, %g7 - bne,pn %icc, BACKOFF_LABEL(2f, 1b) - nop - retl - nop -2: BACKOFF_SPIN(%o2, %o3, 1b) -ENDPROC(atomic_sub) +#define ATOMIC_OP(op) \ +ENTRY(arch_atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: lduw [%o1], %g1; \ + op %g1, %o0, %g7; \ + cas [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + nop; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(arch_atomic_##op); \ +EXPORT_SYMBOL(arch_atomic_##op); -ENTRY(atomic_add_ret) /* %o0 = increment, %o1 = atomic_ptr */ - BACKOFF_SETUP(%o2) -1: lduw [%o1], %g1 - add %g1, %o0, %g7 - cas [%o1], %g1, %g7 - cmp %g1, %g7 - bne,pn %icc, BACKOFF_LABEL(2f, 1b) - add %g1, %o0, %g1 - retl - sra %g1, 0, %o0 -2: BACKOFF_SPIN(%o2, %o3, 1b) -ENDPROC(atomic_add_ret) +#define ATOMIC_OP_RETURN(op) \ +ENTRY(arch_atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */\ + BACKOFF_SETUP(%o2); \ +1: lduw [%o1], %g1; \ + op %g1, %o0, %g7; \ + cas [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ + op %g1, %o0, %g1; \ + retl; \ + sra %g1, 0, %o0; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(arch_atomic_##op##_return); \ +EXPORT_SYMBOL(arch_atomic_##op##_return); -ENTRY(atomic_sub_ret) /* %o0 = decrement, %o1 = atomic_ptr */ - BACKOFF_SETUP(%o2) -1: lduw [%o1], %g1 - sub %g1, %o0, %g7 - cas [%o1], %g1, %g7 - cmp %g1, %g7 - bne,pn %icc, BACKOFF_LABEL(2f, 1b) - sub %g1, %o0, %g1 - retl - sra %g1, 0, %o0 -2: BACKOFF_SPIN(%o2, %o3, 1b) -ENDPROC(atomic_sub_ret) +#define ATOMIC_FETCH_OP(op) \ +ENTRY(arch_atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: lduw [%o1], %g1; \ + op %g1, %o0, %g7; \ + cas [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + sra %g1, 0, %o0; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(arch_atomic_fetch_##op); \ +EXPORT_SYMBOL(arch_atomic_fetch_##op); -ENTRY(atomic64_add) /* %o0 = increment, %o1 = atomic_ptr */ - BACKOFF_SETUP(%o2) -1: ldx [%o1], %g1 - add %g1, %o0, %g7 - casx [%o1], %g1, %g7 - cmp %g1, %g7 - bne,pn %xcc, BACKOFF_LABEL(2f, 1b) - nop - retl - nop -2: BACKOFF_SPIN(%o2, %o3, 1b) -ENDPROC(atomic64_add) +ATOMIC_OP(add) +ATOMIC_OP_RETURN(add) +ATOMIC_FETCH_OP(add) -ENTRY(atomic64_sub) /* %o0 = decrement, %o1 = atomic_ptr */ - BACKOFF_SETUP(%o2) -1: ldx [%o1], %g1 - sub %g1, %o0, %g7 - casx [%o1], %g1, %g7 - cmp %g1, %g7 - bne,pn %xcc, BACKOFF_LABEL(2f, 1b) - nop - retl - nop -2: BACKOFF_SPIN(%o2, %o3, 1b) -ENDPROC(atomic64_sub) +ATOMIC_OP(sub) +ATOMIC_OP_RETURN(sub) +ATOMIC_FETCH_OP(sub) -ENTRY(atomic64_add_ret) /* %o0 = increment, %o1 = atomic_ptr */ - BACKOFF_SETUP(%o2) -1: ldx [%o1], %g1 - add %g1, %o0, %g7 - casx [%o1], %g1, %g7 - cmp %g1, %g7 - bne,pn %xcc, BACKOFF_LABEL(2f, 1b) - nop - retl - add %g1, %o0, %o0 -2: BACKOFF_SPIN(%o2, %o3, 1b) -ENDPROC(atomic64_add_ret) +ATOMIC_OP(and) +ATOMIC_FETCH_OP(and) -ENTRY(atomic64_sub_ret) /* %o0 = decrement, %o1 = atomic_ptr */ - BACKOFF_SETUP(%o2) -1: ldx [%o1], %g1 - sub %g1, %o0, %g7 - casx [%o1], %g1, %g7 - cmp %g1, %g7 - bne,pn %xcc, BACKOFF_LABEL(2f, 1b) - nop - retl - sub %g1, %o0, %o0 -2: BACKOFF_SPIN(%o2, %o3, 1b) -ENDPROC(atomic64_sub_ret) +ATOMIC_OP(or) +ATOMIC_FETCH_OP(or) + +ATOMIC_OP(xor) +ATOMIC_FETCH_OP(xor) + +#undef ATOMIC_FETCH_OP +#undef ATOMIC_OP_RETURN +#undef ATOMIC_OP + +#define ATOMIC64_OP(op) \ +ENTRY(arch_atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: ldx [%o1], %g1; \ + op %g1, %o0, %g7; \ + casx [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + nop; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(arch_atomic64_##op); \ +EXPORT_SYMBOL(arch_atomic64_##op); + +#define ATOMIC64_OP_RETURN(op) \ +ENTRY(arch_atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: ldx [%o1], %g1; \ + op %g1, %o0, %g7; \ + casx [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + op %g1, %o0, %o0; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(arch_atomic64_##op##_return); \ +EXPORT_SYMBOL(arch_atomic64_##op##_return); + +#define ATOMIC64_FETCH_OP(op) \ +ENTRY(arch_atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: ldx [%o1], %g1; \ + op %g1, %o0, %g7; \ + casx [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + mov %g1, %o0; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(arch_atomic64_fetch_##op); \ +EXPORT_SYMBOL(arch_atomic64_fetch_##op); + +ATOMIC64_OP(add) +ATOMIC64_OP_RETURN(add) +ATOMIC64_FETCH_OP(add) + +ATOMIC64_OP(sub) +ATOMIC64_OP_RETURN(sub) +ATOMIC64_FETCH_OP(sub) + +ATOMIC64_OP(and) +ATOMIC64_FETCH_OP(and) + +ATOMIC64_OP(or) +ATOMIC64_FETCH_OP(or) + +ATOMIC64_OP(xor) +ATOMIC64_FETCH_OP(xor) + +#undef ATOMIC64_FETCH_OP +#undef ATOMIC64_OP_RETURN +#undef ATOMIC64_OP -ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */ +ENTRY(arch_atomic64_dec_if_positive) /* %o0 = atomic_ptr */ BACKOFF_SETUP(%o2) 1: ldx [%o0], %g1 brlez,pn %g1, 3f @@ -130,4 +162,5 @@ ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */ 3: retl sub %g1, 1, %o0 2: BACKOFF_SPIN(%o2, %o3, 1b) -ENDPROC(atomic64_dec_if_positive) +ENDPROC(arch_atomic64_dec_if_positive) +EXPORT_SYMBOL(arch_atomic64_dec_if_positive) diff --git a/arch/sparc/lib/bitext.c b/arch/sparc/lib/bitext.c index 8ec4e9c0251a..32a5c1d9459c 100644 --- a/arch/sparc/lib/bitext.c +++ b/arch/sparc/lib/bitext.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * bitext.c: kernel little helper (of bit shuffling variety). * diff --git a/arch/sparc/lib/bitops.S b/arch/sparc/lib/bitops.S index 36f72cc0e67e..9c91cbb310e7 100644 --- a/arch/sparc/lib/bitops.S +++ b/arch/sparc/lib/bitops.S @@ -1,8 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* bitops.S: Sparc64 atomic bit operations. * * Copyright (C) 2000, 2007 David S. Miller (davem@davemloft.net) */ +#include <linux/export.h> #include <linux/linkage.h> #include <asm/asi.h> #include <asm/backoff.h> @@ -29,6 +31,7 @@ ENTRY(test_and_set_bit) /* %o0=nr, %o1=addr */ nop 2: BACKOFF_SPIN(%o3, %o4, 1b) ENDPROC(test_and_set_bit) +EXPORT_SYMBOL(test_and_set_bit) ENTRY(test_and_clear_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) @@ -50,6 +53,7 @@ ENTRY(test_and_clear_bit) /* %o0=nr, %o1=addr */ nop 2: BACKOFF_SPIN(%o3, %o4, 1b) ENDPROC(test_and_clear_bit) +EXPORT_SYMBOL(test_and_clear_bit) ENTRY(test_and_change_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) @@ -71,6 +75,7 @@ ENTRY(test_and_change_bit) /* %o0=nr, %o1=addr */ nop 2: BACKOFF_SPIN(%o3, %o4, 1b) ENDPROC(test_and_change_bit) +EXPORT_SYMBOL(test_and_change_bit) ENTRY(set_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) @@ -90,6 +95,7 @@ ENTRY(set_bit) /* %o0=nr, %o1=addr */ nop 2: BACKOFF_SPIN(%o3, %o4, 1b) ENDPROC(set_bit) +EXPORT_SYMBOL(set_bit) ENTRY(clear_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) @@ -109,6 +115,7 @@ ENTRY(clear_bit) /* %o0=nr, %o1=addr */ nop 2: BACKOFF_SPIN(%o3, %o4, 1b) ENDPROC(clear_bit) +EXPORT_SYMBOL(clear_bit) ENTRY(change_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) @@ -128,3 +135,4 @@ ENTRY(change_bit) /* %o0=nr, %o1=addr */ nop 2: BACKOFF_SPIN(%o3, %o4, 1b) ENDPROC(change_bit) +EXPORT_SYMBOL(change_bit) diff --git a/arch/sparc/lib/blockops.S b/arch/sparc/lib/blockops.S index 3c771011ff4b..5b92959a4d48 100644 --- a/arch/sparc/lib/blockops.S +++ b/arch/sparc/lib/blockops.S @@ -1,9 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * blockops.S: Common block zero optimized routines. * * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) */ +#include <linux/export.h> #include <linux/linkage.h> #include <asm/page.h> @@ -64,6 +66,7 @@ ENTRY(bzero_1page) retl nop ENDPROC(bzero_1page) +EXPORT_SYMBOL(bzero_1page) ENTRY(__copy_1page) /* NOTE: If you change the number of insns of this routine, please check @@ -87,3 +90,4 @@ ENTRY(__copy_1page) retl nop ENDPROC(__copy_1page) +EXPORT_SYMBOL(__copy_1page) diff --git a/arch/sparc/lib/bzero.S b/arch/sparc/lib/bzero.S index 8c058114b649..2bfa44a6b25e 100644 --- a/arch/sparc/lib/bzero.S +++ b/arch/sparc/lib/bzero.S @@ -1,9 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* bzero.S: Simple prefetching memset, bzero, and clear_user * implementations. * * Copyright (C) 2005 David S. Miller <davem@davemloft.net> */ +#include <linux/export.h> #include <linux/linkage.h> .text @@ -78,6 +80,8 @@ __bzero_done: mov %o3, %o0 ENDPROC(__bzero) ENDPROC(memset) +EXPORT_SYMBOL(__bzero) +EXPORT_SYMBOL(memset) #define EX_ST(x,y) \ 98: x,y; \ @@ -143,3 +147,4 @@ __clear_user_done: retl clr %o0 ENDPROC(__clear_user) +EXPORT_SYMBOL(__clear_user) diff --git a/arch/sparc/lib/checksum_32.S b/arch/sparc/lib/checksum_32.S index 0084c3361e15..66eda40fce36 100644 --- a/arch/sparc/lib/checksum_32.S +++ b/arch/sparc/lib/checksum_32.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* checksum.S: Sparc optimized checksum code. * * Copyright(C) 1995 Linus Torvalds @@ -13,6 +14,7 @@ * BSD4.4 portable checksum routine */ +#include <linux/export.h> #include <asm/errno.h> #define CSUM_BIGCHUNK(buf, offset, sum, t0, t1, t2, t3, t4, t5) \ @@ -104,6 +106,7 @@ csum_partial_fix_alignment: * buffer of size 0x20. Follow the code path for that case. */ .globl csum_partial + EXPORT_SYMBOL(csum_partial) csum_partial: /* %o0=buf, %o1=len, %o2=sum */ andcc %o0, 0x7, %g0 ! alignment problems? bne csum_partial_fix_alignment ! yep, handle it @@ -141,44 +144,14 @@ cpte: bne csum_partial_end_cruft ! yep, handle it cpout: retl ! get outta here mov %o2, %o0 ! return computed csum - .globl __csum_partial_copy_start, __csum_partial_copy_end -__csum_partial_copy_start: - /* Work around cpp -rob */ #define ALLOC #alloc #define EXECINSTR #execinstr -#define EX(x,y,a,b) \ -98: x,y; \ - .section .fixup,ALLOC,EXECINSTR; \ - .align 4; \ -99: ba 30f; \ - a, b, %o3; \ - .section __ex_table,ALLOC; \ - .align 4; \ - .word 98b, 99b; \ - .text; \ - .align 4 - -#define EX2(x,y) \ -98: x,y; \ - .section __ex_table,ALLOC; \ - .align 4; \ - .word 98b, 30f; \ - .text; \ - .align 4 - -#define EX3(x,y) \ +#define EX(x,y) \ 98: x,y; \ .section __ex_table,ALLOC; \ .align 4; \ - .word 98b, 96f; \ - .text; \ - .align 4 - -#define EXT(start,end,handler) \ - .section __ex_table,ALLOC; \ - .align 4; \ - .word start, 0, end, handler; \ + .word 98b, cc_fault; \ .text; \ .align 4 @@ -189,20 +162,20 @@ __csum_partial_copy_start: * please check the fixup code below as well. */ #define CSUMCOPY_BIGCHUNK_ALIGNED(src, dst, sum, off, t0, t1, t2, t3, t4, t5, t6, t7) \ - ldd [src + off + 0x00], t0; \ - ldd [src + off + 0x08], t2; \ + EX(ldd [src + off + 0x00], t0); \ + EX(ldd [src + off + 0x08], t2); \ addxcc t0, sum, sum; \ - ldd [src + off + 0x10], t4; \ + EX(ldd [src + off + 0x10], t4); \ addxcc t1, sum, sum; \ - ldd [src + off + 0x18], t6; \ + EX(ldd [src + off + 0x18], t6); \ addxcc t2, sum, sum; \ - std t0, [dst + off + 0x00]; \ + EX(std t0, [dst + off + 0x00]); \ addxcc t3, sum, sum; \ - std t2, [dst + off + 0x08]; \ + EX(std t2, [dst + off + 0x08]); \ addxcc t4, sum, sum; \ - std t4, [dst + off + 0x10]; \ + EX(std t4, [dst + off + 0x10]); \ addxcc t5, sum, sum; \ - std t6, [dst + off + 0x18]; \ + EX(std t6, [dst + off + 0x18]); \ addxcc t6, sum, sum; \ addxcc t7, sum, sum; @@ -211,59 +184,59 @@ __csum_partial_copy_start: * Viking MXCC into streaming mode. Ho hum... */ #define CSUMCOPY_BIGCHUNK(src, dst, sum, off, t0, t1, t2, t3, t4, t5, t6, t7) \ - ldd [src + off + 0x00], t0; \ - ldd [src + off + 0x08], t2; \ - ldd [src + off + 0x10], t4; \ - ldd [src + off + 0x18], t6; \ - st t0, [dst + off + 0x00]; \ + EX(ldd [src + off + 0x00], t0); \ + EX(ldd [src + off + 0x08], t2); \ + EX(ldd [src + off + 0x10], t4); \ + EX(ldd [src + off + 0x18], t6); \ + EX(st t0, [dst + off + 0x00]); \ addxcc t0, sum, sum; \ - st t1, [dst + off + 0x04]; \ + EX(st t1, [dst + off + 0x04]); \ addxcc t1, sum, sum; \ - st t2, [dst + off + 0x08]; \ + EX(st t2, [dst + off + 0x08]); \ addxcc t2, sum, sum; \ - st t3, [dst + off + 0x0c]; \ + EX(st t3, [dst + off + 0x0c]); \ addxcc t3, sum, sum; \ - st t4, [dst + off + 0x10]; \ + EX(st t4, [dst + off + 0x10]); \ addxcc t4, sum, sum; \ - st t5, [dst + off + 0x14]; \ + EX(st t5, [dst + off + 0x14]); \ addxcc t5, sum, sum; \ - st t6, [dst + off + 0x18]; \ + EX(st t6, [dst + off + 0x18]); \ addxcc t6, sum, sum; \ - st t7, [dst + off + 0x1c]; \ + EX(st t7, [dst + off + 0x1c]); \ addxcc t7, sum, sum; /* Yuck, 6 superscalar cycles... */ #define CSUMCOPY_LASTCHUNK(src, dst, sum, off, t0, t1, t2, t3) \ - ldd [src - off - 0x08], t0; \ - ldd [src - off - 0x00], t2; \ + EX(ldd [src - off - 0x08], t0); \ + EX(ldd [src - off - 0x00], t2); \ addxcc t0, sum, sum; \ - st t0, [dst - off - 0x08]; \ + EX(st t0, [dst - off - 0x08]); \ addxcc t1, sum, sum; \ - st t1, [dst - off - 0x04]; \ + EX(st t1, [dst - off - 0x04]); \ addxcc t2, sum, sum; \ - st t2, [dst - off - 0x00]; \ + EX(st t2, [dst - off - 0x00]); \ addxcc t3, sum, sum; \ - st t3, [dst - off + 0x04]; + EX(st t3, [dst - off + 0x04]); /* Handle the end cruft code out of band for better cache patterns. */ cc_end_cruft: be 1f andcc %o3, 4, %g0 - EX(ldd [%o0 + 0x00], %g2, and %o3, 0xf) + EX(ldd [%o0 + 0x00], %g2) add %o1, 8, %o1 addcc %g2, %g7, %g7 add %o0, 8, %o0 addxcc %g3, %g7, %g7 - EX2(st %g2, [%o1 - 0x08]) + EX(st %g2, [%o1 - 0x08]) addx %g0, %g7, %g7 andcc %o3, 4, %g0 - EX2(st %g3, [%o1 - 0x04]) + EX(st %g3, [%o1 - 0x04]) 1: be 1f andcc %o3, 3, %o3 - EX(ld [%o0 + 0x00], %g2, add %o3, 4) + EX(ld [%o0 + 0x00], %g2) add %o1, 4, %o1 addcc %g2, %g7, %g7 - EX2(st %g2, [%o1 - 0x04]) + EX(st %g2, [%o1 - 0x04]) addx %g0, %g7, %g7 andcc %o3, 3, %g0 add %o0, 4, %o0 @@ -273,14 +246,14 @@ cc_end_cruft: subcc %o3, 2, %o3 b 4f or %g0, %g0, %o4 -2: EX(lduh [%o0 + 0x00], %o4, add %o3, 2) +2: EX(lduh [%o0 + 0x00], %o4) add %o0, 2, %o0 - EX2(sth %o4, [%o1 + 0x00]) + EX(sth %o4, [%o1 + 0x00]) be 6f add %o1, 2, %o1 sll %o4, 16, %o4 -4: EX(ldub [%o0 + 0x00], %o5, add %g0, 1) - EX2(stb %o5, [%o1 + 0x00]) +4: EX(ldub [%o0 + 0x00], %o5) + EX(stb %o5, [%o1 + 0x00]) sll %o5, 8, %o5 or %o5, %o4, %o4 6: addcc %o4, %g7, %g7 @@ -303,9 +276,9 @@ cc_dword_align: andcc %o0, 0x2, %g0 be 1f andcc %o0, 0x4, %g0 - EX(lduh [%o0 + 0x00], %g4, add %g1, 0) + EX(lduh [%o0 + 0x00], %g4) sub %g1, 2, %g1 - EX2(sth %g4, [%o1 + 0x00]) + EX(sth %g4, [%o1 + 0x00]) add %o0, 2, %o0 sll %g4, 16, %g4 addcc %g4, %g7, %g7 @@ -319,9 +292,9 @@ cc_dword_align: or %g3, %g7, %g7 1: be 3f andcc %g1, 0xffffff80, %g0 - EX(ld [%o0 + 0x00], %g4, add %g1, 0) + EX(ld [%o0 + 0x00], %g4) sub %g1, 4, %g1 - EX2(st %g4, [%o1 + 0x00]) + EX(st %g4, [%o1 + 0x00]) add %o0, 4, %o0 addcc %g4, %g7, %g7 add %o1, 4, %o1 @@ -335,6 +308,7 @@ cc_dword_align: */ .align 8 .globl __csum_partial_copy_sparc_generic + EXPORT_SYMBOL(__csum_partial_copy_sparc_generic) __csum_partial_copy_sparc_generic: /* %o0=src, %o1=dest, %g1=len, %g7=sum */ xor %o0, %o1, %o4 ! get changing bits @@ -350,7 +324,6 @@ __csum_partial_copy_sparc_generic: CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x20,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x40,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x60,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) -10: EXT(5b, 10b, 20f) ! note for exception handling sub %g1, 128, %g1 ! detract from length addx %g0, %g7, %g7 ! add in last carry bit andcc %g1, 0xffffff80, %g0 ! more to csum? @@ -375,8 +348,7 @@ cctbl: CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x68,%g2,%g3,%g4,%g5) CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x28,%g2,%g3,%g4,%g5) CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x18,%g2,%g3,%g4,%g5) CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x08,%g2,%g3,%g4,%g5) -12: EXT(cctbl, 12b, 22f) ! note for exception table handling - addx %g0, %g7, %g7 +12: addx %g0, %g7, %g7 andcc %o3, 0xf, %g0 ! check for low bits set ccte: bne cc_end_cruft ! something left, handle it out of band andcc %o3, 8, %g0 ! begin checks for that code @@ -386,7 +358,6 @@ ccdbl: CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x00,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x20,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x40,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x60,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) -11: EXT(ccdbl, 11b, 21f) ! note for exception table handling sub %g1, 128, %g1 ! detract from length addx %g0, %g7, %g7 ! add in last carry bit andcc %g1, 0xffffff80, %g0 ! more to csum? @@ -403,9 +374,9 @@ ccslow: cmp %g1, 0 be,a 1f srl %g1, 1, %g4 sub %g1, 1, %g1 - EX(ldub [%o0], %g5, add %g1, 1) + EX(ldub [%o0], %g5) add %o0, 1, %o0 - EX2(stb %g5, [%o1]) + EX(stb %g5, [%o1]) srl %g1, 1, %g4 add %o1, 1, %o1 1: cmp %g4, 0 @@ -414,34 +385,34 @@ ccslow: cmp %g1, 0 andcc %o0, 2, %g0 be,a 1f srl %g4, 1, %g4 - EX(lduh [%o0], %o4, add %g1, 0) + EX(lduh [%o0], %o4) sub %g1, 2, %g1 srl %o4, 8, %g2 sub %g4, 1, %g4 - EX2(stb %g2, [%o1]) + EX(stb %g2, [%o1]) add %o4, %g5, %g5 - EX2(stb %o4, [%o1 + 1]) + EX(stb %o4, [%o1 + 1]) add %o0, 2, %o0 srl %g4, 1, %g4 add %o1, 2, %o1 1: cmp %g4, 0 be,a 2f andcc %g1, 2, %g0 - EX3(ld [%o0], %o4) + EX(ld [%o0], %o4) 5: srl %o4, 24, %g2 srl %o4, 16, %g3 - EX2(stb %g2, [%o1]) + EX(stb %g2, [%o1]) srl %o4, 8, %g2 - EX2(stb %g3, [%o1 + 1]) + EX(stb %g3, [%o1 + 1]) add %o0, 4, %o0 - EX2(stb %g2, [%o1 + 2]) + EX(stb %g2, [%o1 + 2]) addcc %o4, %g5, %g5 - EX2(stb %o4, [%o1 + 3]) + EX(stb %o4, [%o1 + 3]) addx %g5, %g0, %g5 ! I am now to lazy to optimize this (question it add %o1, 4, %o1 ! is worthy). Maybe some day - with the sll/srl subcc %g4, 1, %g4 ! tricks bne,a 5b - EX3(ld [%o0], %o4) + EX(ld [%o0], %o4) sll %g5, 16, %g2 srl %g5, 16, %g5 srl %g2, 16, %g2 @@ -449,19 +420,19 @@ ccslow: cmp %g1, 0 add %g2, %g5, %g5 2: be,a 3f andcc %g1, 1, %g0 - EX(lduh [%o0], %o4, and %g1, 3) + EX(lduh [%o0], %o4) andcc %g1, 1, %g0 srl %o4, 8, %g2 add %o0, 2, %o0 - EX2(stb %g2, [%o1]) + EX(stb %g2, [%o1]) add %g5, %o4, %g5 - EX2(stb %o4, [%o1 + 1]) + EX(stb %o4, [%o1 + 1]) add %o1, 2, %o1 3: be,a 1f sll %g5, 16, %o4 - EX(ldub [%o0], %g2, add %g0, 1) + EX(ldub [%o0], %g2) sll %g2, 8, %o4 - EX2(stb %g2, [%o1]) + EX(stb %g2, [%o1]) add %g5, %o4, %g5 sll %g5, 16, %o4 1: addcc %o4, %g5, %g5 @@ -477,113 +448,10 @@ ccslow: cmp %g1, 0 4: addcc %g7, %g5, %g7 retl addx %g0, %g7, %o0 -__csum_partial_copy_end: /* We do these strange calculations for the csum_*_from_user case only, ie. * we only bother with faults on loads... */ -/* o2 = ((g2%20)&3)*8 - * o3 = g1 - (g2/20)*32 - o2 */ -20: - cmp %g2, 20 - blu,a 1f - and %g2, 3, %o2 - sub %g1, 32, %g1 - b 20b - sub %g2, 20, %g2 -1: - sll %o2, 3, %o2 - b 31f - sub %g1, %o2, %o3 - -/* o2 = (!(g2 & 15) ? 0 : (((g2 & 15) + 1) & ~1)*8) - * o3 = g1 - (g2/16)*32 - o2 */ -21: - andcc %g2, 15, %o3 - srl %g2, 4, %g2 - be,a 1f - clr %o2 - add %o3, 1, %o3 - and %o3, 14, %o3 - sll %o3, 3, %o2 -1: - sll %g2, 5, %g2 - sub %g1, %g2, %o3 - b 31f - sub %o3, %o2, %o3 - -/* o0 += (g2/10)*16 - 0x70 - * 01 += (g2/10)*16 - 0x70 - * o2 = (g2 % 10) ? 8 : 0 - * o3 += 0x70 - (g2/10)*16 - o2 */ -22: - cmp %g2, 10 - blu,a 1f - sub %o0, 0x70, %o0 - add %o0, 16, %o0 - add %o1, 16, %o1 - sub %o3, 16, %o3 - b 22b - sub %g2, 10, %g2 -1: - sub %o1, 0x70, %o1 - add %o3, 0x70, %o3 - clr %o2 - tst %g2 - bne,a 1f - mov 8, %o2 -1: - b 31f - sub %o3, %o2, %o3 -96: - and %g1, 3, %g1 - sll %g4, 2, %g4 - add %g1, %g4, %o3 -30: -/* %o1 is dst - * %o3 is # bytes to zero out - * %o4 is faulting address - * %o5 is %pc where fault occurred */ - clr %o2 -31: -/* %o0 is src - * %o1 is dst - * %o2 is # of bytes to copy from src to dst - * %o3 is # bytes to zero out - * %o4 is faulting address - * %o5 is %pc where fault occurred */ - save %sp, -104, %sp - mov %i5, %o0 - mov %i7, %o1 - mov %i4, %o2 - call lookup_fault - mov %g7, %i4 - cmp %o0, 2 - bne 1f - add %g0, -EFAULT, %i5 - tst %i2 - be 2f - mov %i0, %o1 - mov %i1, %o0 -5: - call memcpy - mov %i2, %o2 - tst %o0 - bne,a 2f - add %i3, %i2, %i3 - add %i1, %i2, %i1 -2: - mov %i1, %o0 -6: - call __bzero - mov %i3, %o1 -1: - ld [%sp + 168], %o2 ! struct_ptr of parent - st %i5, [%o2] - ret - restore - - .section __ex_table,#alloc - .align 4 - .word 5b,2 - .word 6b,2 +cc_fault: + retl + clr %o0 diff --git a/arch/sparc/lib/checksum_64.S b/arch/sparc/lib/checksum_64.S index 1d230f693dc4..32b626f3fe4d 100644 --- a/arch/sparc/lib/checksum_64.S +++ b/arch/sparc/lib/checksum_64.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* checksum.S: Sparc V9 optimized checksum code. * * Copyright(C) 1995 Linus Torvalds @@ -13,6 +14,7 @@ * BSD4.4 portable checksum routine */ +#include <linux/export.h> .text csum_partial_fix_alignment: @@ -37,6 +39,8 @@ csum_partial_fix_alignment: .align 32 .globl csum_partial + .type csum_partial,#function + EXPORT_SYMBOL(csum_partial) csum_partial: /* %o0=buff, %o1=len, %o2=sum */ prefetch [%o0 + 0x000], #n_reads clr %o4 diff --git a/arch/sparc/lib/clear_page.S b/arch/sparc/lib/clear_page.S index 77e531f6c2a7..e63458194f5a 100644 --- a/arch/sparc/lib/clear_page.S +++ b/arch/sparc/lib/clear_page.S @@ -1,13 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* clear_page.S: UltraSparc optimized clear page. * * Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com) * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com) */ +#include <linux/export.h> +#include <linux/pgtable.h> #include <asm/visasm.h> #include <asm/thread_info.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/spitfire.h> #include <asm/head.h> @@ -26,6 +28,7 @@ .text .globl _clear_page + EXPORT_SYMBOL(_clear_page) _clear_page: /* %o0=dest */ ba,pt %xcc, clear_page_common clr %o4 @@ -35,12 +38,13 @@ _clear_page: /* %o0=dest */ */ .align 32 .globl clear_user_page + EXPORT_SYMBOL(clear_user_page) clear_user_page: /* %o0=dest, %o1=vaddr */ lduw [%g6 + TI_PRE_COUNT], %o2 - sethi %uhi(PAGE_OFFSET), %g2 + sethi %hi(PAGE_OFFSET), %g2 sethi %hi(PAGE_SIZE), %o4 - sllx %g2, 32, %g2 + ldx [%g2 + %lo(PAGE_OFFSET)], %g2 sethi %hi(PAGE_KERNEL_LOCKED), %g3 ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 diff --git a/arch/sparc/lib/cmpdi2.c b/arch/sparc/lib/cmpdi2.c deleted file mode 100644 index 8c1306437ed1..000000000000 --- a/arch/sparc/lib/cmpdi2.c +++ /dev/null @@ -1,27 +0,0 @@ -#include <linux/module.h> - -#include "libgcc.h" - -word_type __cmpdi2(long long a, long long b) -{ - const DWunion au = { - .ll = a - }; - const DWunion bu = { - .ll = b - }; - - if (au.s.high < bu.s.high) - return 0; - else if (au.s.high > bu.s.high) - return 2; - - if ((unsigned int) au.s.low < (unsigned int) bu.s.low) - return 0; - else if ((unsigned int) au.s.low > (unsigned int) bu.s.low) - return 2; - - return 1; -} - -EXPORT_SYMBOL(__cmpdi2); diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S index 302c0e60dc2c..e23e6a69ff92 100644 --- a/arch/sparc/lib/copy_in_user.S +++ b/arch/sparc/lib/copy_in_user.S @@ -1,25 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* copy_in_user.S: Copy from userspace to userspace. * * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ +#include <linux/export.h> #include <linux/linkage.h> #include <asm/asi.h> #define XCC xcc -#define EX(x,y) \ +#define EX(x,y,z) \ 98: x,y; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, z; \ .text; \ .align 4; +#define EX_O4(x,y) EX(x,y,__retl_o4_plus_8) +#define EX_O2_4(x,y) EX(x,y,__retl_o2_plus_4) +#define EX_O2_1(x,y) EX(x,y,__retl_o2_plus_1) + .register %g2,#scratch .register %g3,#scratch .text +__retl_o4_plus_8: + add %o4, %o2, %o4 + retl + add %o4, 8, %o0 +__retl_o2_plus_4: + retl + add %o2, 4, %o0 +__retl_o2_plus_1: + retl + add %o2, 1, %o0 + .align 32 /* Don't try to get too fancy here, just nice and @@ -28,7 +45,7 @@ * to copy register windows around during thread cloning. */ -ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ +ENTRY(raw_copy_in_user) /* %o0=dst, %o1=src, %o2=len */ cmp %o2, 0 be,pn %XCC, 85f or %o0, %o1, %o3 @@ -44,8 +61,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0x7, %o4 and %o2, 0x7, %o2 1: subcc %o4, 0x8, %o4 - EX(ldxa [%o1] %asi, %o5) - EX(stxa %o5, [%o0] %asi) + EX_O4(ldxa [%o1] %asi, %o5) + EX_O4(stxa %o5, [%o0] %asi) add %o1, 0x8, %o1 bgu,pt %XCC, 1b add %o0, 0x8, %o0 @@ -53,8 +70,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ be,pt %XCC, 1f nop sub %o2, 0x4, %o2 - EX(lduwa [%o1] %asi, %o5) - EX(stwa %o5, [%o0] %asi) + EX_O2_4(lduwa [%o1] %asi, %o5) + EX_O2_4(stwa %o5, [%o0] %asi) add %o1, 0x4, %o1 add %o0, 0x4, %o0 1: cmp %o2, 0 @@ -70,8 +87,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ 82: subcc %o2, 4, %o2 - EX(lduwa [%o1] %asi, %g1) - EX(stwa %g1, [%o0] %asi) + EX_O2_4(lduwa [%o1] %asi, %g1) + EX_O2_4(stwa %g1, [%o0] %asi) add %o1, 4, %o1 bgu,pt %XCC, 82b add %o0, 4, %o0 @@ -82,11 +99,12 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ .align 32 90: subcc %o2, 1, %o2 - EX(lduba [%o1] %asi, %g1) - EX(stba %g1, [%o0] %asi) + EX_O2_1(lduba [%o1] %asi, %g1) + EX_O2_1(stba %g1, [%o0] %asi) add %o1, 1, %o1 bgu,pt %XCC, 90b add %o0, 1, %o0 retl clr %o0 -ENDPROC(___copy_in_user) +ENDPROC(raw_copy_in_user) +EXPORT_SYMBOL(raw_copy_in_user) diff --git a/arch/sparc/lib/copy_page.S b/arch/sparc/lib/copy_page.S index 4d2df328e514..7a041f3ebc58 100644 --- a/arch/sparc/lib/copy_page.S +++ b/arch/sparc/lib/copy_page.S @@ -1,13 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* clear_page.S: UltraSparc optimized copy page. * * Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com) * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com) */ +#include <linux/export.h> #include <asm/visasm.h> #include <asm/thread_info.h> #include <asm/page.h> -#include <asm/pgtable.h> +#include <linux/pgtable.h> #include <asm/spitfire.h> #include <asm/head.h> @@ -44,12 +46,13 @@ .align 32 .globl copy_user_page .type copy_user_page,#function + EXPORT_SYMBOL(copy_user_page) copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ lduw [%g6 + TI_PRE_COUNT], %o4 - sethi %uhi(PAGE_OFFSET), %g2 + sethi %hi(PAGE_OFFSET), %g2 sethi %hi(PAGE_SIZE), %o3 - sllx %g2, 32, %g2 + ldx [%g2 + %lo(PAGE_OFFSET)], %g2 sethi %hi(PAGE_KERNEL_LOCKED), %g3 ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 diff --git a/arch/sparc/lib/copy_user.S b/arch/sparc/lib/copy_user.S index ef095b6c43b1..7bb2ef68881d 100644 --- a/arch/sparc/lib/copy_user.S +++ b/arch/sparc/lib/copy_user.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* copy_user.S: Sparc optimized copy_from_user and copy_to_user code. * * Copyright(C) 1995 Linus Torvalds @@ -11,6 +12,7 @@ * Returns 0 if successful, otherwise count of bytes not copied yet */ +#include <linux/export.h> #include <asm/ptrace.h> #include <asm/asmmacro.h> #include <asm/page.h> @@ -19,98 +21,134 @@ /* Work around cpp -rob */ #define ALLOC #alloc #define EXECINSTR #execinstr + +#define EX_ENTRY(l1, l2) \ + .section __ex_table,ALLOC; \ + .align 4; \ + .word l1, l2; \ + .text; + #define EX(x,y,a,b) \ 98: x,y; \ .section .fixup,ALLOC,EXECINSTR; \ .align 4; \ -99: ba fixupretl; \ - a, b, %g3; \ - .section __ex_table,ALLOC; \ - .align 4; \ - .word 98b, 99b; \ - .text; \ - .align 4 +99: retl; \ + a, b, %o0; \ + EX_ENTRY(98b, 99b) #define EX2(x,y,c,d,e,a,b) \ 98: x,y; \ .section .fixup,ALLOC,EXECINSTR; \ .align 4; \ 99: c, d, e; \ - ba fixupretl; \ - a, b, %g3; \ - .section __ex_table,ALLOC; \ - .align 4; \ - .word 98b, 99b; \ - .text; \ - .align 4 + retl; \ + a, b, %o0; \ + EX_ENTRY(98b, 99b) #define EXO2(x,y) \ 98: x, y; \ - .section __ex_table,ALLOC; \ - .align 4; \ - .word 98b, 97f; \ - .text; \ - .align 4 + EX_ENTRY(98b, 97f) -#define EXT(start,end,handler) \ - .section __ex_table,ALLOC; \ - .align 4; \ - .word start, 0, end, handler; \ - .text; \ - .align 4 +#define LD(insn, src, offset, reg, label) \ +98: insn [%src + (offset)], %reg; \ + .section .fixup,ALLOC,EXECINSTR; \ +99: ba label; \ + mov offset, %g5; \ + EX_ENTRY(98b, 99b) -/* Please do not change following macros unless you change logic used - * in .fixup at the end of this file as well - */ +#define ST(insn, dst, offset, reg, label) \ +98: insn %reg, [%dst + (offset)]; \ + .section .fixup,ALLOC,EXECINSTR; \ +99: ba label; \ + mov offset, %g5; \ + EX_ENTRY(98b, 99b) /* Both these macros have to start with exactly the same insn */ +/* left: g7 + (g1 % 128) - offset */ #define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \ - ldd [%src + (offset) + 0x00], %t0; \ - ldd [%src + (offset) + 0x08], %t2; \ - ldd [%src + (offset) + 0x10], %t4; \ - ldd [%src + (offset) + 0x18], %t6; \ - st %t0, [%dst + (offset) + 0x00]; \ - st %t1, [%dst + (offset) + 0x04]; \ - st %t2, [%dst + (offset) + 0x08]; \ - st %t3, [%dst + (offset) + 0x0c]; \ - st %t4, [%dst + (offset) + 0x10]; \ - st %t5, [%dst + (offset) + 0x14]; \ - st %t6, [%dst + (offset) + 0x18]; \ - st %t7, [%dst + (offset) + 0x1c]; - + LD(ldd, src, offset + 0x00, t0, bigchunk_fault) \ + LD(ldd, src, offset + 0x08, t2, bigchunk_fault) \ + LD(ldd, src, offset + 0x10, t4, bigchunk_fault) \ + LD(ldd, src, offset + 0x18, t6, bigchunk_fault) \ + ST(st, dst, offset + 0x00, t0, bigchunk_fault) \ + ST(st, dst, offset + 0x04, t1, bigchunk_fault) \ + ST(st, dst, offset + 0x08, t2, bigchunk_fault) \ + ST(st, dst, offset + 0x0c, t3, bigchunk_fault) \ + ST(st, dst, offset + 0x10, t4, bigchunk_fault) \ + ST(st, dst, offset + 0x14, t5, bigchunk_fault) \ + ST(st, dst, offset + 0x18, t6, bigchunk_fault) \ + ST(st, dst, offset + 0x1c, t7, bigchunk_fault) + +/* left: g7 + (g1 % 128) - offset */ #define MOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \ - ldd [%src + (offset) + 0x00], %t0; \ - ldd [%src + (offset) + 0x08], %t2; \ - ldd [%src + (offset) + 0x10], %t4; \ - ldd [%src + (offset) + 0x18], %t6; \ - std %t0, [%dst + (offset) + 0x00]; \ - std %t2, [%dst + (offset) + 0x08]; \ - std %t4, [%dst + (offset) + 0x10]; \ - std %t6, [%dst + (offset) + 0x18]; + LD(ldd, src, offset + 0x00, t0, bigchunk_fault) \ + LD(ldd, src, offset + 0x08, t2, bigchunk_fault) \ + LD(ldd, src, offset + 0x10, t4, bigchunk_fault) \ + LD(ldd, src, offset + 0x18, t6, bigchunk_fault) \ + ST(std, dst, offset + 0x00, t0, bigchunk_fault) \ + ST(std, dst, offset + 0x08, t2, bigchunk_fault) \ + ST(std, dst, offset + 0x10, t4, bigchunk_fault) \ + ST(std, dst, offset + 0x18, t6, bigchunk_fault) + + .section .fixup,#alloc,#execinstr +bigchunk_fault: + sub %g7, %g5, %o0 + and %g1, 127, %g1 + retl + add %o0, %g1, %o0 +/* left: offset + 16 + (g1 % 16) */ #define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \ - ldd [%src - (offset) - 0x10], %t0; \ - ldd [%src - (offset) - 0x08], %t2; \ - st %t0, [%dst - (offset) - 0x10]; \ - st %t1, [%dst - (offset) - 0x0c]; \ - st %t2, [%dst - (offset) - 0x08]; \ - st %t3, [%dst - (offset) - 0x04]; + LD(ldd, src, -(offset + 0x10), t0, lastchunk_fault) \ + LD(ldd, src, -(offset + 0x08), t2, lastchunk_fault) \ + ST(st, dst, -(offset + 0x10), t0, lastchunk_fault) \ + ST(st, dst, -(offset + 0x0c), t1, lastchunk_fault) \ + ST(st, dst, -(offset + 0x08), t2, lastchunk_fault) \ + ST(st, dst, -(offset + 0x04), t3, lastchunk_fault) -#define MOVE_HALFCHUNK(src, dst, offset, t0, t1, t2, t3) \ - lduh [%src + (offset) + 0x00], %t0; \ - lduh [%src + (offset) + 0x02], %t1; \ - lduh [%src + (offset) + 0x04], %t2; \ - lduh [%src + (offset) + 0x06], %t3; \ - sth %t0, [%dst + (offset) + 0x00]; \ - sth %t1, [%dst + (offset) + 0x02]; \ - sth %t2, [%dst + (offset) + 0x04]; \ - sth %t3, [%dst + (offset) + 0x06]; + .section .fixup,#alloc,#execinstr +lastchunk_fault: + and %g1, 15, %g1 + retl + sub %g1, %g5, %o0 +/* left: o3 + (o2 % 16) - offset */ +#define MOVE_HALFCHUNK(src, dst, offset, t0, t1, t2, t3) \ + LD(lduh, src, offset + 0x00, t0, halfchunk_fault) \ + LD(lduh, src, offset + 0x02, t1, halfchunk_fault) \ + LD(lduh, src, offset + 0x04, t2, halfchunk_fault) \ + LD(lduh, src, offset + 0x06, t3, halfchunk_fault) \ + ST(sth, dst, offset + 0x00, t0, halfchunk_fault) \ + ST(sth, dst, offset + 0x02, t1, halfchunk_fault) \ + ST(sth, dst, offset + 0x04, t2, halfchunk_fault) \ + ST(sth, dst, offset + 0x06, t3, halfchunk_fault) + +/* left: o3 + (o2 % 16) + offset + 2 */ #define MOVE_SHORTCHUNK(src, dst, offset, t0, t1) \ - ldub [%src - (offset) - 0x02], %t0; \ - ldub [%src - (offset) - 0x01], %t1; \ - stb %t0, [%dst - (offset) - 0x02]; \ - stb %t1, [%dst - (offset) - 0x01]; + LD(ldub, src, -(offset + 0x02), t0, halfchunk_fault) \ + LD(ldub, src, -(offset + 0x01), t1, halfchunk_fault) \ + ST(stb, dst, -(offset + 0x02), t0, halfchunk_fault) \ + ST(stb, dst, -(offset + 0x01), t1, halfchunk_fault) + + .section .fixup,#alloc,#execinstr +halfchunk_fault: + and %o2, 15, %o2 + sub %o3, %g5, %o3 + retl + add %o2, %o3, %o0 + +/* left: offset + 2 + (o2 % 2) */ +#define MOVE_LAST_SHORTCHUNK(src, dst, offset, t0, t1) \ + LD(ldub, src, -(offset + 0x02), t0, last_shortchunk_fault) \ + LD(ldub, src, -(offset + 0x01), t1, last_shortchunk_fault) \ + ST(stb, dst, -(offset + 0x02), t0, last_shortchunk_fault) \ + ST(stb, dst, -(offset + 0x01), t1, last_shortchunk_fault) + + .section .fixup,#alloc,#execinstr +last_shortchunk_fault: + and %o2, 1, %o2 + retl + sub %o2, %g5, %o0 .text .align 4 @@ -119,6 +157,7 @@ __copy_user_begin: .globl __copy_user + EXPORT_SYMBOL(__copy_user) dword_align: andcc %o1, 1, %g0 be 4f @@ -179,8 +218,6 @@ __copy_user: /* %o0=dst %o1=src %o2=len */ MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5) MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5) MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5) -80: - EXT(5b, 80b, 50f) subcc %g7, 128, %g7 add %o1, 128, %o1 bne 5b @@ -198,7 +235,6 @@ __copy_user: /* %o0=dst %o1=src %o2=len */ jmpl %o5 + %lo(copy_user_table_end), %g0 add %o0, %g7, %o0 -copy_user_table: MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g4, g5) MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g4, g5) MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g4, g5) @@ -207,7 +243,6 @@ copy_user_table: MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g4, g5) MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g4, g5) copy_user_table_end: - EXT(copy_user_table, copy_user_table_end, 51f) be copy_user_last7 andcc %g1, 4, %g0 @@ -247,8 +282,6 @@ ldd_std: MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5) MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5) MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5) -81: - EXT(ldd_std, 81b, 52f) subcc %g7, 128, %g7 add %o1, 128, %o1 bne ldd_std @@ -287,8 +320,6 @@ cannot_optimize: 10: MOVE_HALFCHUNK(o1, o0, 0x00, g2, g3, g4, g5) MOVE_HALFCHUNK(o1, o0, 0x08, g2, g3, g4, g5) -82: - EXT(10b, 82b, 53f) subcc %o3, 0x10, %o3 add %o1, 0x10, %o1 bne 10b @@ -305,8 +336,6 @@ byte_chunk: MOVE_SHORTCHUNK(o1, o0, -0x0c, g2, g3) MOVE_SHORTCHUNK(o1, o0, -0x0e, g2, g3) MOVE_SHORTCHUNK(o1, o0, -0x10, g2, g3) -83: - EXT(byte_chunk, 83b, 54f) subcc %o3, 0x10, %o3 add %o1, 0x10, %o1 bne byte_chunk @@ -322,16 +351,14 @@ short_end: add %o1, %o3, %o1 jmpl %o5 + %lo(short_table_end), %g0 andcc %o2, 1, %g0 -84: - MOVE_SHORTCHUNK(o1, o0, 0x0c, g2, g3) - MOVE_SHORTCHUNK(o1, o0, 0x0a, g2, g3) - MOVE_SHORTCHUNK(o1, o0, 0x08, g2, g3) - MOVE_SHORTCHUNK(o1, o0, 0x06, g2, g3) - MOVE_SHORTCHUNK(o1, o0, 0x04, g2, g3) - MOVE_SHORTCHUNK(o1, o0, 0x02, g2, g3) - MOVE_SHORTCHUNK(o1, o0, 0x00, g2, g3) + MOVE_LAST_SHORTCHUNK(o1, o0, 0x0c, g2, g3) + MOVE_LAST_SHORTCHUNK(o1, o0, 0x0a, g2, g3) + MOVE_LAST_SHORTCHUNK(o1, o0, 0x08, g2, g3) + MOVE_LAST_SHORTCHUNK(o1, o0, 0x06, g2, g3) + MOVE_LAST_SHORTCHUNK(o1, o0, 0x04, g2, g3) + MOVE_LAST_SHORTCHUNK(o1, o0, 0x02, g2, g3) + MOVE_LAST_SHORTCHUNK(o1, o0, 0x00, g2, g3) short_table_end: - EXT(84b, short_table_end, 55f) be 1f nop EX(ldub [%o1], %g2, add %g0, 1) @@ -360,137 +387,8 @@ short_aligned_end: .section .fixup,#alloc,#execinstr .align 4 97: - mov %o2, %g3 -fixupretl: - sethi %hi(PAGE_OFFSET), %g1 - cmp %o0, %g1 - blu 1f - cmp %o1, %g1 - bgeu 1f - ld [%g6 + TI_PREEMPT], %g1 - cmp %g1, 0 - bne 1f - nop - save %sp, -64, %sp - mov %i0, %o0 - call __bzero - mov %g3, %o1 - restore -1: retl - mov %g3, %o0 - -/* exception routine sets %g2 to (broken_insn - first_insn)>>2 */ -50: -/* This magic counts how many bytes are left when crash in MOVE_BIGCHUNK - * happens. This is derived from the amount ldd reads, st stores, etc. - * x = g2 % 12; - * g3 = g1 + g7 - ((g2 / 12) * 32 + (x < 4) ? 0 : (x - 4) * 4); - * o0 += (g2 / 12) * 32; - */ - cmp %g2, 12 - add %o0, %g7, %o0 - bcs 1f - cmp %g2, 24 - bcs 2f - cmp %g2, 36 - bcs 3f - nop - sub %g2, 12, %g2 - sub %g7, 32, %g7 -3: sub %g2, 12, %g2 - sub %g7, 32, %g7 -2: sub %g2, 12, %g2 - sub %g7, 32, %g7 -1: cmp %g2, 4 - bcs,a 60f - clr %g2 - sub %g2, 4, %g2 - sll %g2, 2, %g2 -60: and %g1, 0x7f, %g3 - sub %o0, %g7, %o0 - add %g3, %g7, %g3 - ba fixupretl - sub %g3, %g2, %g3 -51: -/* i = 41 - g2; j = i % 6; - * g3 = (g1 & 15) + (i / 6) * 16 + (j < 4) ? (j + 1) * 4 : 16; - * o0 -= (i / 6) * 16 + 16; - */ - neg %g2 - and %g1, 0xf, %g1 - add %g2, 41, %g2 - add %o0, %g1, %o0 -1: cmp %g2, 6 - bcs,a 2f - cmp %g2, 4 - add %g1, 16, %g1 - b 1b - sub %g2, 6, %g2 -2: bcc,a 2f - mov 16, %g2 - inc %g2 - sll %g2, 2, %g2 -2: add %g1, %g2, %g3 - ba fixupretl - sub %o0, %g3, %o0 -52: -/* g3 = g1 + g7 - (g2 / 8) * 32 + (g2 & 4) ? (g2 & 3) * 8 : 0; - o0 += (g2 / 8) * 32 */ - andn %g2, 7, %g4 - add %o0, %g7, %o0 - andcc %g2, 4, %g0 - and %g2, 3, %g2 - sll %g4, 2, %g4 - sll %g2, 3, %g2 - bne 60b - sub %g7, %g4, %g7 - ba 60b - clr %g2 -53: -/* g3 = o3 + (o2 & 15) - (g2 & 8) - (g2 & 4) ? (g2 & 3) * 2 : 0; - o0 += (g2 & 8) */ - and %g2, 3, %g4 - andcc %g2, 4, %g0 - and %g2, 8, %g2 - sll %g4, 1, %g4 - be 1f - add %o0, %g2, %o0 - add %g2, %g4, %g2 -1: and %o2, 0xf, %g3 - add %g3, %o3, %g3 - ba fixupretl - sub %g3, %g2, %g3 -54: -/* g3 = o3 + (o2 & 15) - (g2 / 4) * 2 - (g2 & 2) ? (g2 & 1) : 0; - o0 += (g2 / 4) * 2 */ - srl %g2, 2, %o4 - and %g2, 1, %o5 - srl %g2, 1, %g2 - add %o4, %o4, %o4 - and %o5, %g2, %o5 - and %o2, 0xf, %o2 - add %o0, %o4, %o0 - sub %o3, %o5, %o3 - sub %o2, %o4, %o2 - ba fixupretl - add %o2, %o3, %g3 -55: -/* i = 27 - g2; - g3 = (o2 & 1) + i / 4 * 2 + !(i & 3); - o0 -= i / 4 * 2 + 1 */ - neg %g2 - and %o2, 1, %o2 - add %g2, 27, %g2 - srl %g2, 2, %o5 - andcc %g2, 3, %g0 - mov 1, %g2 - add %o5, %o5, %o5 - be,a 1f - clr %g2 -1: add %g2, %o5, %g3 - sub %o0, %g3, %o0 - ba fixupretl - add %g3, %o2, %g3 + retl + mov %o2, %o0 .globl __copy_user_end __copy_user_end: diff --git a/arch/sparc/lib/csum_copy.S b/arch/sparc/lib/csum_copy.S index e566c770a0f6..f968e83bc93b 100644 --- a/arch/sparc/lib/csum_copy.S +++ b/arch/sparc/lib/csum_copy.S @@ -1,8 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* csum_copy.S: Checksum+copy code for sparc64 * * Copyright (C) 2005 David S. Miller <davem@davemloft.net> */ +#include <linux/export.h> + #ifdef __KERNEL__ #define GLOBAL_SPARE %g7 #else @@ -63,9 +66,12 @@ add %o5, %o4, %o4 .globl FUNC_NAME -FUNC_NAME: /* %o0=src, %o1=dst, %o2=len, %o3=sum */ + .type FUNC_NAME,#function + EXPORT_SYMBOL(FUNC_NAME) +FUNC_NAME: /* %o0=src, %o1=dst, %o2=len */ LOAD(prefetch, %o0 + 0x000, #n_reads) xor %o0, %o1, %g1 + mov -1, %o3 clr %o4 andcc %g1, 0x3, %g0 bne,pn %icc, 95f diff --git a/arch/sparc/lib/csum_copy_from_user.S b/arch/sparc/lib/csum_copy_from_user.S index e0304e6a2242..b0ba8d4dd439 100644 --- a/arch/sparc/lib/csum_copy_from_user.S +++ b/arch/sparc/lib/csum_copy_from_user.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* csum_copy_from_user.S: Checksum+copy from userspace. * * Copyright (C) 2005 David S. Miller (davem@davemloft.net) @@ -8,14 +9,14 @@ .section .fixup, "ax"; \ .align 4; \ 99: retl; \ - mov -1, %o0; \ + mov 0, %o0; \ .section __ex_table,"a";\ .align 4; \ .word 98b, 99b; \ .text; \ .align 4; -#define FUNC_NAME __csum_partial_copy_from_user +#define FUNC_NAME csum_and_copy_from_user #define LOAD(type,addr,dest) type##a [addr] %asi, dest #include "csum_copy.S" diff --git a/arch/sparc/lib/csum_copy_to_user.S b/arch/sparc/lib/csum_copy_to_user.S index afd01acc587c..91ba36dbf7d2 100644 --- a/arch/sparc/lib/csum_copy_to_user.S +++ b/arch/sparc/lib/csum_copy_to_user.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* csum_copy_to_user.S: Checksum+copy to userspace. * * Copyright (C) 2005 David S. Miller (davem@davemloft.net) @@ -8,14 +9,14 @@ .section .fixup,"ax"; \ .align 4; \ 99: retl; \ - mov -1, %o0; \ + mov 0, %o0; \ .section __ex_table,"a";\ .align 4; \ .word 98b, 99b; \ .text; \ .align 4; -#define FUNC_NAME __csum_partial_copy_to_user +#define FUNC_NAME csum_and_copy_to_user #define STORE(type,src,addr) type##a src, [addr] %asi #include "csum_copy.S" diff --git a/arch/sparc/lib/divdi3.S b/arch/sparc/lib/divdi3.S index 9614b48b6ef8..4ba901acd572 100644 --- a/arch/sparc/lib/divdi3.S +++ b/arch/sparc/lib/divdi3.S @@ -1,22 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* Copyright (C) 1989, 1992, 1993, 1994, 1995 Free Software Foundation, Inc. This file is part of GNU CC. -GNU CC is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2, or (at your option) -any later version. - -GNU CC is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with GNU CC; see the file COPYING. If not, write to -the Free Software Foundation, 59 Temple Place - Suite 330, -Boston, MA 02111-1307, USA. */ + */ +#include <linux/export.h> .text .align 4 .globl __divdi3 @@ -279,3 +268,4 @@ __divdi3: .LL81: ret restore +EXPORT_SYMBOL(__divdi3) diff --git a/arch/sparc/lib/ffs.S b/arch/sparc/lib/ffs.S index b39389f69899..3a9ad8ffdfe8 100644 --- a/arch/sparc/lib/ffs.S +++ b/arch/sparc/lib/ffs.S @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <linux/export.h> #include <linux/linkage.h> .register %g2,#scratch @@ -65,6 +67,8 @@ ENTRY(__ffs) add %o2, %g1, %o0 ENDPROC(ffs) ENDPROC(__ffs) +EXPORT_SYMBOL(__ffs) +EXPORT_SYMBOL(ffs) .section .popc_6insn_patch, "ax" .word ffs diff --git a/arch/sparc/lib/fls.S b/arch/sparc/lib/fls.S new file mode 100644 index 000000000000..ccf97fb7d8cd --- /dev/null +++ b/arch/sparc/lib/fls.S @@ -0,0 +1,67 @@ +/* fls.S: SPARC default fls definition. + * + * SPARC default fls definition, which follows the same algorithm as + * in generic fls(). This function will be boot time patched on T4 + * and onward. + */ + +#include <linux/export.h> +#include <linux/linkage.h> + + .text + .register %g2, #scratch + .register %g3, #scratch +ENTRY(fls) + brz,pn %o0, 6f + mov 0, %o1 + sethi %hi(0xffff0000), %g3 + mov %o0, %g2 + andcc %o0, %g3, %g0 + be,pt %icc, 8f + mov 32, %o1 + sethi %hi(0xff000000), %g3 + andcc %g2, %g3, %g0 + bne,pt %icc, 3f + sethi %hi(0xf0000000), %g3 + sll %o0, 8, %o0 +1: + add %o1, -8, %o1 + sra %o0, 0, %o0 + mov %o0, %g2 +2: + sethi %hi(0xf0000000), %g3 +3: + andcc %g2, %g3, %g0 + bne,pt %icc, 4f + sethi %hi(0xc0000000), %g3 + sll %o0, 4, %o0 + add %o1, -4, %o1 + sra %o0, 0, %o0 + mov %o0, %g2 +4: + andcc %g2, %g3, %g0 + be,a,pt %icc, 7f + sll %o0, 2, %o0 +5: + xnor %g0, %o0, %o0 + srl %o0, 31, %o0 + sub %o1, %o0, %o1 +6: + jmp %o7 + 8 + sra %o1, 0, %o0 +7: + add %o1, -2, %o1 + ba,pt %xcc, 5b + sra %o0, 0, %o0 +8: + sll %o0, 16, %o0 + sethi %hi(0xff000000), %g3 + sra %o0, 0, %o0 + mov %o0, %g2 + andcc %g2, %g3, %g0 + bne,pt %icc, 2b + mov 16, %o1 + ba,pt %xcc, 1b + sll %o0, 8, %o0 +ENDPROC(fls) +EXPORT_SYMBOL(fls) diff --git a/arch/sparc/lib/fls64.S b/arch/sparc/lib/fls64.S new file mode 100644 index 000000000000..87005b67d378 --- /dev/null +++ b/arch/sparc/lib/fls64.S @@ -0,0 +1,61 @@ +/* fls64.S: SPARC default __fls definition. + * + * SPARC default __fls definition, which follows the same algorithm as + * in generic __fls(). This function will be boot time patched on T4 + * and onward. + */ + +#include <linux/export.h> +#include <linux/linkage.h> + + .text + .register %g2, #scratch + .register %g3, #scratch +ENTRY(__fls) + mov -1, %g2 + sllx %g2, 32, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 1f + mov 63, %g1 + sllx %o0, 32, %o0 + mov 31, %g1 +1: + mov -1, %g2 + sllx %g2, 48, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 2f + mov -1, %g2 + sllx %o0, 16, %o0 + add %g1, -16, %g1 +2: + mov -1, %g2 + sllx %g2, 56, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 3f + mov -1, %g2 + sllx %o0, 8, %o0 + add %g1, -8, %g1 +3: + sllx %g2, 60, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 4f + mov -1, %g2 + sllx %o0, 4, %o0 + add %g1, -4, %g1 +4: + sllx %g2, 62, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 5f + mov -1, %g3 + sllx %o0, 2, %o0 + add %g1, -2, %g1 +5: + mov 0, %g2 + sllx %g3, 63, %g3 + and %o0, %g3, %o0 + movre %o0, 1, %g2 + sub %g1, %g2, %g1 + jmp %o7+8 + sra %g1, 0, %o0 +ENDPROC(__fls) +EXPORT_SYMBOL(__fls) diff --git a/arch/sparc/lib/hweight.S b/arch/sparc/lib/hweight.S index 95414e0a6808..eebee59b0655 100644 --- a/arch/sparc/lib/hweight.S +++ b/arch/sparc/lib/hweight.S @@ -1,12 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <linux/export.h> #include <linux/linkage.h> .text .align 32 ENTRY(__arch_hweight8) - ba,pt %xcc, __sw_hweight8 + sethi %hi(__sw_hweight8), %g1 + jmpl %g1 + %lo(__sw_hweight8), %g0 nop - nop ENDPROC(__arch_hweight8) +EXPORT_SYMBOL(__arch_hweight8) .section .popc_3insn_patch, "ax" .word __arch_hweight8 sllx %o0, 64-8, %g1 @@ -15,10 +18,11 @@ ENDPROC(__arch_hweight8) .previous ENTRY(__arch_hweight16) - ba,pt %xcc, __sw_hweight16 + sethi %hi(__sw_hweight16), %g1 + jmpl %g1 + %lo(__sw_hweight16), %g0 nop - nop ENDPROC(__arch_hweight16) +EXPORT_SYMBOL(__arch_hweight16) .section .popc_3insn_patch, "ax" .word __arch_hweight16 sllx %o0, 64-16, %g1 @@ -27,10 +31,11 @@ ENDPROC(__arch_hweight16) .previous ENTRY(__arch_hweight32) - ba,pt %xcc, __sw_hweight32 + sethi %hi(__sw_hweight32), %g1 + jmpl %g1 + %lo(__sw_hweight32), %g0 nop - nop ENDPROC(__arch_hweight32) +EXPORT_SYMBOL(__arch_hweight32) .section .popc_3insn_patch, "ax" .word __arch_hweight32 sllx %o0, 64-32, %g1 @@ -39,10 +44,11 @@ ENDPROC(__arch_hweight32) .previous ENTRY(__arch_hweight64) - ba,pt %xcc, __sw_hweight64 + sethi %hi(__sw_hweight64), %g1 + jmpl %g1 + %lo(__sw_hweight64), %g0 nop - nop ENDPROC(__arch_hweight64) +EXPORT_SYMBOL(__arch_hweight64) .section .popc_3insn_patch, "ax" .word __arch_hweight64 retl diff --git a/arch/sparc/lib/iomap.c b/arch/sparc/lib/iomap.c index c4d42a50ebc0..f3a8cd491ce0 100644 --- a/arch/sparc/lib/iomap.c +++ b/arch/sparc/lib/iomap.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Implement the sparc iomap interfaces */ @@ -18,8 +19,10 @@ void ioport_unmap(void __iomem *addr) EXPORT_SYMBOL(ioport_map); EXPORT_SYMBOL(ioport_unmap); +#ifdef CONFIG_PCI void pci_iounmap(struct pci_dev *dev, void __iomem * addr) { /* nothing to do */ } EXPORT_SYMBOL(pci_iounmap); +#endif diff --git a/arch/sparc/lib/ipcsum.S b/arch/sparc/lib/ipcsum.S index 4742d59029ee..7fa8fd4b795a 100644 --- a/arch/sparc/lib/ipcsum.S +++ b/arch/sparc/lib/ipcsum.S @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <linux/export.h> #include <linux/linkage.h> .text @@ -31,3 +33,4 @@ ENTRY(ip_fast_csum) /* %o0 = iph, %o1 = ihl */ retl and %o2, %o1, %o0 ENDPROC(ip_fast_csum) +EXPORT_SYMBOL(ip_fast_csum) diff --git a/arch/sparc/lib/ksyms.c b/arch/sparc/lib/ksyms.c deleted file mode 100644 index 0c4e35e522fa..000000000000 --- a/arch/sparc/lib/ksyms.c +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Export of symbols defined in assembler - */ - -/* Tell string.h we don't want memcpy etc. as cpp defines */ -#define EXPORT_SYMTAB_STROPS - -#include <linux/module.h> -#include <linux/string.h> -#include <linux/types.h> - -#include <asm/checksum.h> -#include <asm/uaccess.h> -#include <asm/ftrace.h> - -/* string functions */ -EXPORT_SYMBOL(strlen); -EXPORT_SYMBOL(strncmp); - -/* mem* functions */ -extern void *__memscan_zero(void *, size_t); -extern void *__memscan_generic(void *, int, size_t); -extern void *__bzero(void *, size_t); - -EXPORT_SYMBOL(memscan); -EXPORT_SYMBOL(__memscan_zero); -EXPORT_SYMBOL(__memscan_generic); -EXPORT_SYMBOL(memcmp); -EXPORT_SYMBOL(memcpy); -EXPORT_SYMBOL(memset); -EXPORT_SYMBOL(memmove); -EXPORT_SYMBOL(__bzero); - -/* Networking helper routines. */ -EXPORT_SYMBOL(csum_partial); - -#ifdef CONFIG_MCOUNT -EXPORT_SYMBOL(_mcount); -#endif - -/* - * sparc - */ -#ifdef CONFIG_SPARC32 -extern int __ashrdi3(int, int); -extern int __ashldi3(int, int); -extern int __lshrdi3(int, int); -extern int __muldi3(int, int); -extern int __divdi3(int, int); - -extern void (*__copy_1page)(void *, const void *); -extern void (*bzero_1page)(void *); - -extern void ___rw_read_enter(void); -extern void ___rw_read_try(void); -extern void ___rw_read_exit(void); -extern void ___rw_write_enter(void); - -/* Networking helper routines. */ -EXPORT_SYMBOL(__csum_partial_copy_sparc_generic); - -/* Special internal versions of library functions. */ -EXPORT_SYMBOL(__copy_1page); -EXPORT_SYMBOL(__memmove); -EXPORT_SYMBOL(bzero_1page); - -/* Moving data to/from/in userspace. */ -EXPORT_SYMBOL(__copy_user); - -/* Used by asm/spinlock.h */ -#ifdef CONFIG_SMP -EXPORT_SYMBOL(___rw_read_enter); -EXPORT_SYMBOL(___rw_read_try); -EXPORT_SYMBOL(___rw_read_exit); -EXPORT_SYMBOL(___rw_write_enter); -#endif - -EXPORT_SYMBOL(__ashrdi3); -EXPORT_SYMBOL(__ashldi3); -EXPORT_SYMBOL(__lshrdi3); -EXPORT_SYMBOL(__muldi3); -EXPORT_SYMBOL(__divdi3); -#endif - -/* - * sparc64 - */ -#ifdef CONFIG_SPARC64 -/* Networking helper routines. */ -EXPORT_SYMBOL(csum_partial_copy_nocheck); -EXPORT_SYMBOL(__csum_partial_copy_from_user); -EXPORT_SYMBOL(__csum_partial_copy_to_user); -EXPORT_SYMBOL(ip_fast_csum); - -/* Moving data to/from/in userspace. */ -EXPORT_SYMBOL(___copy_to_user); -EXPORT_SYMBOL(___copy_from_user); -EXPORT_SYMBOL(___copy_in_user); -EXPORT_SYMBOL(__clear_user); - -/* RW semaphores */ -EXPORT_SYMBOL(__down_read); -EXPORT_SYMBOL(__down_read_trylock); -EXPORT_SYMBOL(__down_write); -EXPORT_SYMBOL(__down_write_trylock); -EXPORT_SYMBOL(__up_read); -EXPORT_SYMBOL(__up_write); -EXPORT_SYMBOL(__downgrade_write); - -/* Atomic counter implementation. */ -EXPORT_SYMBOL(atomic_add); -EXPORT_SYMBOL(atomic_add_ret); -EXPORT_SYMBOL(atomic_sub); -EXPORT_SYMBOL(atomic_sub_ret); -EXPORT_SYMBOL(atomic64_add); -EXPORT_SYMBOL(atomic64_add_ret); -EXPORT_SYMBOL(atomic64_sub); -EXPORT_SYMBOL(atomic64_sub_ret); -EXPORT_SYMBOL(atomic64_dec_if_positive); - -/* Atomic bit operations. */ -EXPORT_SYMBOL(test_and_set_bit); -EXPORT_SYMBOL(test_and_clear_bit); -EXPORT_SYMBOL(test_and_change_bit); -EXPORT_SYMBOL(set_bit); -EXPORT_SYMBOL(clear_bit); -EXPORT_SYMBOL(change_bit); - -/* Special internal versions of library functions. */ -EXPORT_SYMBOL(_clear_page); -EXPORT_SYMBOL(clear_user_page); -EXPORT_SYMBOL(copy_user_page); - -/* RAID code needs this */ -void VISenter(void); -EXPORT_SYMBOL(VISenter); - -/* CRYPTO code needs this */ -void VISenterhalf(void); -EXPORT_SYMBOL(VISenterhalf); - -extern void xor_vis_2(unsigned long, unsigned long *, unsigned long *); -extern void xor_vis_3(unsigned long, unsigned long *, unsigned long *, - unsigned long *); -extern void xor_vis_4(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *); -extern void xor_vis_5(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *, unsigned long *); -EXPORT_SYMBOL(xor_vis_2); -EXPORT_SYMBOL(xor_vis_3); -EXPORT_SYMBOL(xor_vis_4); -EXPORT_SYMBOL(xor_vis_5); - -extern void xor_niagara_2(unsigned long, unsigned long *, unsigned long *); -extern void xor_niagara_3(unsigned long, unsigned long *, unsigned long *, - unsigned long *); -extern void xor_niagara_4(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *); -extern void xor_niagara_5(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *, unsigned long *); - -EXPORT_SYMBOL(xor_niagara_2); -EXPORT_SYMBOL(xor_niagara_3); -EXPORT_SYMBOL(xor_niagara_4); -EXPORT_SYMBOL(xor_niagara_5); -#endif diff --git a/arch/sparc/lib/libgcc.h b/arch/sparc/lib/libgcc.h index b84fd797f3ea..79845c941b87 100644 --- a/arch/sparc/lib/libgcc.h +++ b/arch/sparc/lib/libgcc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ASM_LIBGCC_H #define __ASM_LIBGCC_H diff --git a/arch/sparc/lib/locks.S b/arch/sparc/lib/locks.S index 64f53f2b673d..47a39f4384a2 100644 --- a/arch/sparc/lib/locks.S +++ b/arch/sparc/lib/locks.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * locks.S: SMP low-level lock primitives on Sparc. * @@ -6,6 +7,7 @@ * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) */ +#include <linux/export.h> #include <asm/ptrace.h> #include <asm/psr.h> #include <asm/smp.h> @@ -48,6 +50,7 @@ ___rw_write_enter_spin_on_wlock: ld [%g1], %g2 .globl ___rw_read_enter +EXPORT_SYMBOL(___rw_read_enter) ___rw_read_enter: orcc %g2, 0x0, %g0 bne,a ___rw_read_enter_spin_on_wlock @@ -59,6 +62,7 @@ ___rw_read_enter: mov %g4, %o7 .globl ___rw_read_exit +EXPORT_SYMBOL(___rw_read_exit) ___rw_read_exit: orcc %g2, 0x0, %g0 bne,a ___rw_read_exit_spin_on_wlock @@ -70,6 +74,7 @@ ___rw_read_exit: mov %g4, %o7 .globl ___rw_read_try +EXPORT_SYMBOL(___rw_read_try) ___rw_read_try: orcc %g2, 0x0, %g0 bne ___rw_read_try_spin_on_wlock @@ -81,6 +86,7 @@ ___rw_read_try: mov %g4, %o7 .globl ___rw_write_enter +EXPORT_SYMBOL(___rw_write_enter) ___rw_write_enter: orcc %g2, 0x0, %g0 bne ___rw_write_enter_spin_on_wlock diff --git a/arch/sparc/lib/lshrdi3.S b/arch/sparc/lib/lshrdi3.S index 60ebc7cdbee0..09bf581a0ba5 100644 --- a/arch/sparc/lib/lshrdi3.S +++ b/arch/sparc/lib/lshrdi3.S @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <linux/export.h> #include <linux/linkage.h> ENTRY(__lshrdi3) @@ -25,3 +27,4 @@ ENTRY(__lshrdi3) retl nop ENDPROC(__lshrdi3) +EXPORT_SYMBOL(__lshrdi3) diff --git a/arch/sparc/lib/mcount.S b/arch/sparc/lib/mcount.S index 3ad6cbdc2163..f7f7910eb41e 100644 --- a/arch/sparc/lib/mcount.S +++ b/arch/sparc/lib/mcount.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2000 Anton Blanchard (anton@linuxcare.com) * @@ -5,6 +6,7 @@ * This can also be tweaked for kernel stack overflow detection. */ +#include <linux/export.h> #include <linux/linkage.h> /* @@ -16,6 +18,7 @@ .align 32 .globl _mcount .type _mcount,#function + EXPORT_SYMBOL(_mcount) .globl mcount .type mcount,#function _mcount: @@ -24,10 +27,7 @@ mcount: #ifdef CONFIG_DYNAMIC_FTRACE /* Do nothing, the retl/nop below is all we need. */ #else - sethi %hi(function_trace_stop), %g1 - lduw [%g1 + %lo(function_trace_stop)], %g2 - brnz,pn %g2, 2f - sethi %hi(ftrace_trace_function), %g1 + sethi %hi(ftrace_trace_function), %g1 sethi %hi(ftrace_stub), %g2 ldx [%g1 + %lo(ftrace_trace_function)], %g1 or %g2, %lo(ftrace_stub), %g2 @@ -80,11 +80,8 @@ ftrace_stub: .globl ftrace_caller .type ftrace_caller,#function ftrace_caller: - sethi %hi(function_trace_stop), %g1 mov %i7, %g2 - lduw [%g1 + %lo(function_trace_stop)], %g1 - brnz,pn %g1, ftrace_stub - mov %fp, %g3 + mov %fp, %g3 save %sp, -176, %sp mov %g2, %o1 mov %g2, %l0 diff --git a/arch/sparc/lib/memcmp.S b/arch/sparc/lib/memcmp.S index efa106c41ed0..c87e8000feba 100644 --- a/arch/sparc/lib/memcmp.S +++ b/arch/sparc/lib/memcmp.S @@ -1,9 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Sparc optimized memcmp code. * * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) * Copyright (C) 2000, 2008 David S. Miller (davem@davemloft.net) */ +#include <linux/export.h> #include <linux/linkage.h> #include <asm/asm.h> @@ -25,3 +27,4 @@ ENTRY(memcmp) 2: retl mov 0, %o0 ENDPROC(memcmp) +EXPORT_SYMBOL(memcmp) diff --git a/arch/sparc/lib/memcpy.S b/arch/sparc/lib/memcpy.S index 4d8c497517bd..57b1ae0f5924 100644 --- a/arch/sparc/lib/memcpy.S +++ b/arch/sparc/lib/memcpy.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* memcpy.S: Sparc optimized memcpy and memmove code * Hand optimized from GNU libc's memcpy and memmove * Copyright (C) 1991,1996 Free Software Foundation @@ -7,6 +8,8 @@ * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ +#include <linux/export.h> + #define FUNC(x) \ .globl x; \ .type x,@function; \ @@ -58,93 +61,11 @@ x: stb %t0, [%dst - (offset) - 0x02]; \ stb %t1, [%dst - (offset) - 0x01]; -/* Both these macros have to start with exactly the same insn */ -#define RMOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \ - ldd [%src - (offset) - 0x20], %t0; \ - ldd [%src - (offset) - 0x18], %t2; \ - ldd [%src - (offset) - 0x10], %t4; \ - ldd [%src - (offset) - 0x08], %t6; \ - st %t0, [%dst - (offset) - 0x20]; \ - st %t1, [%dst - (offset) - 0x1c]; \ - st %t2, [%dst - (offset) - 0x18]; \ - st %t3, [%dst - (offset) - 0x14]; \ - st %t4, [%dst - (offset) - 0x10]; \ - st %t5, [%dst - (offset) - 0x0c]; \ - st %t6, [%dst - (offset) - 0x08]; \ - st %t7, [%dst - (offset) - 0x04]; - -#define RMOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \ - ldd [%src - (offset) - 0x20], %t0; \ - ldd [%src - (offset) - 0x18], %t2; \ - ldd [%src - (offset) - 0x10], %t4; \ - ldd [%src - (offset) - 0x08], %t6; \ - std %t0, [%dst - (offset) - 0x20]; \ - std %t2, [%dst - (offset) - 0x18]; \ - std %t4, [%dst - (offset) - 0x10]; \ - std %t6, [%dst - (offset) - 0x08]; - -#define RMOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \ - ldd [%src + (offset) + 0x00], %t0; \ - ldd [%src + (offset) + 0x08], %t2; \ - st %t0, [%dst + (offset) + 0x00]; \ - st %t1, [%dst + (offset) + 0x04]; \ - st %t2, [%dst + (offset) + 0x08]; \ - st %t3, [%dst + (offset) + 0x0c]; - -#define RMOVE_SHORTCHUNK(src, dst, offset, t0, t1) \ - ldub [%src + (offset) + 0x00], %t0; \ - ldub [%src + (offset) + 0x01], %t1; \ - stb %t0, [%dst + (offset) + 0x00]; \ - stb %t1, [%dst + (offset) + 0x01]; - -#define SMOVE_CHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, prev, shil, shir, offset2) \ - ldd [%src + (offset) + 0x00], %t0; \ - ldd [%src + (offset) + 0x08], %t2; \ - srl %t0, shir, %t5; \ - srl %t1, shir, %t6; \ - sll %t0, shil, %t0; \ - or %t5, %prev, %t5; \ - sll %t1, shil, %prev; \ - or %t6, %t0, %t0; \ - srl %t2, shir, %t1; \ - srl %t3, shir, %t6; \ - sll %t2, shil, %t2; \ - or %t1, %prev, %t1; \ - std %t4, [%dst + (offset) + (offset2) - 0x04]; \ - std %t0, [%dst + (offset) + (offset2) + 0x04]; \ - sll %t3, shil, %prev; \ - or %t6, %t2, %t4; - -#define SMOVE_ALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, prev, shil, shir, offset2) \ - ldd [%src + (offset) + 0x00], %t0; \ - ldd [%src + (offset) + 0x08], %t2; \ - srl %t0, shir, %t4; \ - srl %t1, shir, %t5; \ - sll %t0, shil, %t6; \ - or %t4, %prev, %t0; \ - sll %t1, shil, %prev; \ - or %t5, %t6, %t1; \ - srl %t2, shir, %t4; \ - srl %t3, shir, %t5; \ - sll %t2, shil, %t6; \ - or %t4, %prev, %t2; \ - sll %t3, shil, %prev; \ - or %t5, %t6, %t3; \ - std %t0, [%dst + (offset) + (offset2) + 0x00]; \ - std %t2, [%dst + (offset) + (offset2) + 0x08]; - .text .align 4 -0: - retl - nop ! Only bcopy returns here and it retuns void... - -#ifdef __KERNEL__ -FUNC(amemmove) -FUNC(__memmove) -#endif FUNC(memmove) +EXPORT_SYMBOL(memmove) cmp %o0, %o1 mov %o0, %g7 bleu 9f @@ -202,6 +123,7 @@ FUNC(memmove) add %o0, 2, %o0 FUNC(memcpy) /* %o0=dst %o1=src %o2=len */ +EXPORT_SYMBOL(memcpy) sub %o0, %o1, %o4 mov %o0, %g7 diff --git a/arch/sparc/lib/memmove.S b/arch/sparc/lib/memmove.S index b7f6334e159f..543dda7b9dac 100644 --- a/arch/sparc/lib/memmove.S +++ b/arch/sparc/lib/memmove.S @@ -1,16 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* memmove.S: Simple memmove implementation. * * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com) * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz) */ +#include <linux/export.h> #include <linux/linkage.h> .text ENTRY(memmove) /* o0=dst o1=src o2=len */ - mov %o0, %g1 + brz,pn %o2, 99f + mov %o0, %g1 + cmp %o0, %o1 - bleu,pt %xcc, memcpy + bleu,pt %xcc, 2f add %o1, %o2, %g7 cmp %g7, %o0 bleu,pt %xcc, memcpy @@ -24,7 +28,35 @@ ENTRY(memmove) /* o0=dst o1=src o2=len */ stb %g7, [%o0] bne,pt %icc, 1b sub %o0, 1, %o0 - +99: retl mov %g1, %o0 + + /* We can't just call memcpy for these memmove cases. On some + * chips the memcpy uses cache initializing stores and when dst + * and src are close enough, those can clobber the source data + * before we've loaded it in. + */ +2: or %o0, %o1, %g7 + or %o2, %g7, %g7 + andcc %g7, 0x7, %g0 + bne,pn %xcc, 4f + nop + +3: ldx [%o1], %g7 + add %o1, 8, %o1 + subcc %o2, 8, %o2 + add %o0, 8, %o0 + bne,pt %icc, 3b + stx %g7, [%o0 - 0x8] + ba,a,pt %xcc, 99b + +4: ldub [%o1], %g7 + add %o1, 1, %o1 + subcc %o2, 1, %o2 + add %o0, 1, %o0 + bne,pt %icc, 4b + stb %g7, [%o0 - 0x1] + ba,a,pt %xcc, 99b ENDPROC(memmove) +EXPORT_SYMBOL(memmove) diff --git a/arch/sparc/lib/memscan_32.S b/arch/sparc/lib/memscan_32.S index 4ff1657dfc24..5386a3a20019 100644 --- a/arch/sparc/lib/memscan_32.S +++ b/arch/sparc/lib/memscan_32.S @@ -1,9 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * memscan.S: Optimized memscan for the Sparc. * * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) */ +#include <linux/export.h> + /* In essence, this is just a fancy strlen. */ #define LO_MAGIC 0x01010101 @@ -13,6 +16,8 @@ .align 4 .globl __memscan_zero, __memscan_generic .globl memscan +EXPORT_SYMBOL(__memscan_zero) +EXPORT_SYMBOL(__memscan_generic) __memscan_zero: /* %o0 = addr, %o1 = size */ cmp %o1, 0 diff --git a/arch/sparc/lib/memscan_64.S b/arch/sparc/lib/memscan_64.S index 5686dfa5dc15..70a4f21057f2 100644 --- a/arch/sparc/lib/memscan_64.S +++ b/arch/sparc/lib/memscan_64.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * memscan.S: Optimized memscan for Sparc64. * @@ -5,6 +6,8 @@ * Copyright (C) 1998 David S. Miller (davem@redhat.com) */ +#include <linux/export.h> + #define HI_MAGIC 0x8080808080808080 #define LO_MAGIC 0x0101010101010101 #define ASI_PL 0x88 @@ -12,7 +15,11 @@ .text .align 32 .globl __memscan_zero, __memscan_generic + .type __memscan_zero,#function + .type __memscan_generic,#function .globl memscan + EXPORT_SYMBOL(__memscan_zero) + EXPORT_SYMBOL(__memscan_generic) __memscan_zero: /* %o0 = bufp, %o1 = size */ diff --git a/arch/sparc/lib/memset.S b/arch/sparc/lib/memset.S index 99c017be8719..a33419dbb464 100644 --- a/arch/sparc/lib/memset.S +++ b/arch/sparc/lib/memset.S @@ -1,12 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* linux/arch/sparc/lib/memset.S: Sparc optimized memset, bzero and clear_user code * Copyright (C) 1991,1996 Free Software Foundation * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) * - * Returns 0, if ok, and number of bytes not yet set if exception - * occurs and we were called as clear_user. + * Calls to memset returns initial %o0. Calls to bzero returns 0, if ok, and + * number of bytes not yet set if exception occurs and we were called as + * clear_user. */ +#include <linux/export.h> #include <asm/ptrace.h> /* Work around cpp -rob */ @@ -16,7 +19,7 @@ 98: x,y; \ .section .fixup,ALLOC,EXECINSTR; \ .align 4; \ -99: ba 30f; \ +99: retl; \ a, b, %o0; \ .section __ex_table,ALLOC; \ .align 4; \ @@ -24,35 +27,44 @@ .text; \ .align 4 -#define EXT(start,end,handler) \ +#define STORE(source, base, offset, n) \ +98: std source, [base + offset + n]; \ + .section .fixup,ALLOC,EXECINSTR; \ + .align 4; \ +99: ba 30f; \ + sub %o3, n - offset, %o3; \ .section __ex_table,ALLOC; \ .align 4; \ - .word start, 0, end, handler; \ + .word 98b, 99b; \ .text; \ - .align 4 + .align 4; + +#define STORE_LAST(source, base, offset, n) \ + EX(std source, [base - offset - n], \ + add %o1, offset + n); /* Please don't change these macros, unless you change the logic * in the .fixup section below as well. * Store 64 bytes at (BASE + OFFSET) using value SOURCE. */ -#define ZERO_BIG_BLOCK(base, offset, source) \ - std source, [base + offset + 0x00]; \ - std source, [base + offset + 0x08]; \ - std source, [base + offset + 0x10]; \ - std source, [base + offset + 0x18]; \ - std source, [base + offset + 0x20]; \ - std source, [base + offset + 0x28]; \ - std source, [base + offset + 0x30]; \ - std source, [base + offset + 0x38]; +#define ZERO_BIG_BLOCK(base, offset, source) \ + STORE(source, base, offset, 0x00); \ + STORE(source, base, offset, 0x08); \ + STORE(source, base, offset, 0x10); \ + STORE(source, base, offset, 0x18); \ + STORE(source, base, offset, 0x20); \ + STORE(source, base, offset, 0x28); \ + STORE(source, base, offset, 0x30); \ + STORE(source, base, offset, 0x38); #define ZERO_LAST_BLOCKS(base, offset, source) \ - std source, [base - offset - 0x38]; \ - std source, [base - offset - 0x30]; \ - std source, [base - offset - 0x28]; \ - std source, [base - offset - 0x20]; \ - std source, [base - offset - 0x18]; \ - std source, [base - offset - 0x10]; \ - std source, [base - offset - 0x08]; \ - std source, [base - offset - 0x00]; + STORE_LAST(source, base, offset, 0x38); \ + STORE_LAST(source, base, offset, 0x30); \ + STORE_LAST(source, base, offset, 0x28); \ + STORE_LAST(source, base, offset, 0x20); \ + STORE_LAST(source, base, offset, 0x18); \ + STORE_LAST(source, base, offset, 0x10); \ + STORE_LAST(source, base, offset, 0x08); \ + STORE_LAST(source, base, offset, 0x00); .text .align 4 @@ -61,10 +73,13 @@ __bzero_begin: .globl __bzero + .type __bzero,#function .globl memset - .globl __memset_start, __memset_end -__memset_start: + EXPORT_SYMBOL(__bzero) + EXPORT_SYMBOL(memset) memset: + mov %o0, %g1 + mov 1, %g4 and %o1, 0xff, %g3 sll %g3, 8, %g2 or %g3, %g2, %g3 @@ -89,6 +104,7 @@ memset: sub %o0, %o2, %o0 __bzero: + clr %g4 mov %g0, %g3 1: cmp %o1, 7 @@ -113,8 +129,6 @@ __bzero: ZERO_BIG_BLOCK(%o0, 0x00, %g2) subcc %o3, 128, %o3 ZERO_BIG_BLOCK(%o0, 0x40, %g2) -11: - EXT(10b, 11b, 20f) bne 10b add %o0, 128, %o0 @@ -129,7 +143,6 @@ __bzero: jmp %o4 add %o0, %o2, %o0 -12: ZERO_LAST_BLOCKS(%o0, 0x48, %g2) ZERO_LAST_BLOCKS(%o0, 0x08, %g2) 13: @@ -151,8 +164,8 @@ __bzero: bne,a 8f EX(stb %g3, [%o0], and %o1, 1) 8: - retl - clr %o0 + b 0f + nop 7: be 13b orcc %o1, 0, %g0 @@ -164,39 +177,21 @@ __bzero: bne 8b EX(stb %g3, [%o0 - 1], add %o1, 1) 0: + andcc %g4, 1, %g0 + be 5f + nop + retl + mov %g1, %o0 +5: retl clr %o0 -__memset_end: .section .fixup,#alloc,#execinstr .align 4 -20: - cmp %g2, 8 - bleu 1f - and %o1, 0x7f, %o1 - sub %g2, 9, %g2 - add %o3, 64, %o3 -1: - sll %g2, 3, %g2 - add %o3, %o1, %o0 - b 30f - sub %o0, %g2, %o0 -21: - mov 8, %o0 - and %o1, 7, %o1 - sub %o0, %g2, %o0 - sll %o0, 3, %o0 - b 30f - add %o0, %o1, %o0 30: -/* %o4 is faulting address, %o5 is %pc where fault occurred */ - save %sp, -104, %sp - mov %i5, %o0 - mov %i7, %o1 - call lookup_fault - mov %i4, %o2 - ret - restore + and %o1, 0x7f, %o1 + retl + add %o3, %o1, %o0 .globl __bzero_end __bzero_end: diff --git a/arch/sparc/lib/muldi3.S b/arch/sparc/lib/muldi3.S index 9794939d1c12..7e1e8cd30a22 100644 --- a/arch/sparc/lib/muldi3.S +++ b/arch/sparc/lib/muldi3.S @@ -1,22 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* Copyright (C) 1989, 1992, 1993, 1994, 1995 Free Software Foundation, Inc. This file is part of GNU CC. -GNU CC is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2, or (at your option) -any later version. - -GNU CC is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with GNU CC; see the file COPYING. If not, write to -the Free Software Foundation, 59 Temple Place - Suite 330, -Boston, MA 02111-1307, USA. */ + */ +#include <linux/export.h> .text .align 4 .globl __muldi3 @@ -74,3 +63,4 @@ __muldi3: add %l2, %l0, %i0 ret restore %g0, %l3, %o1 +EXPORT_SYMBOL(__muldi3) diff --git a/arch/sparc/lib/multi3.S b/arch/sparc/lib/multi3.S new file mode 100644 index 000000000000..5bb4c122a2cf --- /dev/null +++ b/arch/sparc/lib/multi3.S @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <linux/export.h> +#include <linux/linkage.h> + + .text + .align 4 +ENTRY(__multi3) /* %o0 = u, %o1 = v */ + mov %o1, %g1 + srl %o3, 0, %o4 + mulx %o4, %g1, %o1 + srlx %g1, 0x20, %g3 + mulx %g3, %o4, %g7 + sllx %g7, 0x20, %o5 + srl %g1, 0, %o4 + sub %o1, %o5, %o5 + srlx %o5, 0x20, %o5 + addcc %g7, %o5, %g7 + srlx %o3, 0x20, %o5 + mulx %o4, %o5, %o4 + mulx %g3, %o5, %o5 + sethi %hi(0x80000000), %g3 + addcc %g7, %o4, %g7 + srlx %g7, 0x20, %g7 + add %g3, %g3, %g3 + movcc %xcc, %g0, %g3 + addcc %o5, %g7, %o5 + sllx %o4, 0x20, %o4 + add %o1, %o4, %o1 + add %o5, %g3, %g2 + mulx %g1, %o2, %g1 + add %g1, %g2, %g1 + mulx %o0, %o3, %o0 + retl + add %g1, %o0, %o0 +ENDPROC(__multi3) +EXPORT_SYMBOL(__multi3) diff --git a/arch/sparc/lib/strlen.S b/arch/sparc/lib/strlen.S index 536f83507fbf..27478b3f1647 100644 --- a/arch/sparc/lib/strlen.S +++ b/arch/sparc/lib/strlen.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* strlen.S: Sparc optimized strlen code * Hand optimized from GNU libc's strlen * Copyright (C) 1991,1996 Free Software Foundation @@ -5,6 +6,7 @@ * Copyright (C) 1996, 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ +#include <linux/export.h> #include <linux/linkage.h> #include <asm/asm.h> @@ -78,3 +80,4 @@ ENTRY(strlen) retl mov 2, %o0 ENDPROC(strlen) +EXPORT_SYMBOL(strlen) diff --git a/arch/sparc/lib/strncmp_32.S b/arch/sparc/lib/strncmp_32.S index c0d1b568c1c5..387bbf621548 100644 --- a/arch/sparc/lib/strncmp_32.S +++ b/arch/sparc/lib/strncmp_32.S @@ -1,8 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * strncmp.S: Hand optimized Sparc assembly of GCC output from GNU libc * generic strncmp routine. */ +#include <linux/export.h> #include <linux/linkage.h> .text @@ -116,3 +118,4 @@ ENTRY(strncmp) retl sub %o3, %o0, %o0 ENDPROC(strncmp) +EXPORT_SYMBOL(strncmp) diff --git a/arch/sparc/lib/strncmp_64.S b/arch/sparc/lib/strncmp_64.S index 0656627166f3..76c1207ecf5a 100644 --- a/arch/sparc/lib/strncmp_64.S +++ b/arch/sparc/lib/strncmp_64.S @@ -1,9 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Sparc64 optimized strncmp code. * * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ +#include <linux/export.h> #include <linux/linkage.h> #include <asm/asi.h> @@ -28,3 +30,4 @@ ENTRY(strncmp) retl clr %o0 ENDPROC(strncmp) +EXPORT_SYMBOL(strncmp) diff --git a/arch/sparc/lib/ucmpdi2.c b/arch/sparc/lib/ucmpdi2.c deleted file mode 100644 index 1e06ed500682..000000000000 --- a/arch/sparc/lib/ucmpdi2.c +++ /dev/null @@ -1,19 +0,0 @@ -#include <linux/module.h> -#include "libgcc.h" - -word_type __ucmpdi2(unsigned long long a, unsigned long long b) -{ - const DWunion au = {.ll = a}; - const DWunion bu = {.ll = b}; - - if ((unsigned int) au.s.high < (unsigned int) bu.s.high) - return 0; - else if ((unsigned int) au.s.high > (unsigned int) bu.s.high) - return 2; - if ((unsigned int) au.s.low < (unsigned int) bu.s.low) - return 0; - else if ((unsigned int) au.s.low > (unsigned int) bu.s.low) - return 2; - return 1; -} -EXPORT_SYMBOL(__ucmpdi2); diff --git a/arch/sparc/lib/udivdi3.S b/arch/sparc/lib/udivdi3.S index 24e0a355e2e8..7a1117ec7696 100644 --- a/arch/sparc/lib/udivdi3.S +++ b/arch/sparc/lib/udivdi3.S @@ -1,21 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ /* Copyright (C) 1989, 1992, 1993, 1994, 1995 Free Software Foundation, Inc. This file is part of GNU CC. -GNU CC is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2, or (at your option) -any later version. - -GNU CC is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with GNU CC; see the file COPYING. If not, write to -the Free Software Foundation, 59 Temple Place - Suite 330, -Boston, MA 02111-1307, USA. */ + */ .text .align 4 diff --git a/arch/sparc/lib/user_fixup.c b/arch/sparc/lib/user_fixup.c deleted file mode 100644 index ac96ae236709..000000000000 --- a/arch/sparc/lib/user_fixup.c +++ /dev/null @@ -1,71 +0,0 @@ -/* user_fixup.c: Fix up user copy faults. - * - * Copyright (C) 2004 David S. Miller <davem@redhat.com> - */ - -#include <linux/compiler.h> -#include <linux/kernel.h> -#include <linux/string.h> -#include <linux/errno.h> -#include <linux/module.h> - -#include <asm/uaccess.h> - -/* Calculating the exact fault address when using - * block loads and stores can be very complicated. - * - * Instead of trying to be clever and handling all - * of the cases, just fix things up simply here. - */ - -static unsigned long compute_size(unsigned long start, unsigned long size, unsigned long *offset) -{ - unsigned long fault_addr = current_thread_info()->fault_address; - unsigned long end = start + size; - - if (fault_addr < start || fault_addr >= end) { - *offset = 0; - } else { - *offset = fault_addr - start; - size = end - fault_addr; - } - return size; -} - -unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size) -{ - unsigned long offset; - - size = compute_size((unsigned long) from, size, &offset); - if (likely(size)) - memset(to + offset, 0, size); - - return size; -} -EXPORT_SYMBOL(copy_from_user_fixup); - -unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size) -{ - unsigned long offset; - - return compute_size((unsigned long) to, size, &offset); -} -EXPORT_SYMBOL(copy_to_user_fixup); - -unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size) -{ - unsigned long fault_addr = current_thread_info()->fault_address; - unsigned long start = (unsigned long) to; - unsigned long end = start + size; - - if (fault_addr >= start && fault_addr < end) - return end - fault_addr; - - start = (unsigned long) from; - end = start + size; - if (fault_addr >= start && fault_addr < end) - return end - fault_addr; - - return size; -} -EXPORT_SYMBOL(copy_in_user_fixup); diff --git a/arch/sparc/lib/xor.S b/arch/sparc/lib/xor.S index 2c05641c3263..35461e3b2a9b 100644 --- a/arch/sparc/lib/xor.S +++ b/arch/sparc/lib/xor.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * arch/sparc64/lib/xor.S * @@ -8,6 +9,7 @@ * Copyright (C) 2006 David S. Miller <davem@davemloft.net> */ +#include <linux/export.h> #include <linux/linkage.h> #include <asm/visasm.h> #include <asm/asi.h> @@ -90,6 +92,7 @@ ENTRY(xor_vis_2) retl wr %g0, 0, %fprs ENDPROC(xor_vis_2) +EXPORT_SYMBOL(xor_vis_2) ENTRY(xor_vis_3) rd %fprs, %o5 @@ -156,6 +159,7 @@ ENTRY(xor_vis_3) retl wr %g0, 0, %fprs ENDPROC(xor_vis_3) +EXPORT_SYMBOL(xor_vis_3) ENTRY(xor_vis_4) rd %fprs, %o5 @@ -241,6 +245,7 @@ ENTRY(xor_vis_4) retl wr %g0, 0, %fprs ENDPROC(xor_vis_4) +EXPORT_SYMBOL(xor_vis_4) ENTRY(xor_vis_5) save %sp, -192, %sp @@ -347,6 +352,7 @@ ENTRY(xor_vis_5) ret restore ENDPROC(xor_vis_5) +EXPORT_SYMBOL(xor_vis_5) /* Niagara versions. */ ENTRY(xor_niagara_2) /* %o0=bytes, %o1=dest, %o2=src */ @@ -393,6 +399,7 @@ ENTRY(xor_niagara_2) /* %o0=bytes, %o1=dest, %o2=src */ ret restore ENDPROC(xor_niagara_2) +EXPORT_SYMBOL(xor_niagara_2) ENTRY(xor_niagara_3) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */ save %sp, -192, %sp @@ -454,6 +461,7 @@ ENTRY(xor_niagara_3) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */ ret restore ENDPROC(xor_niagara_3) +EXPORT_SYMBOL(xor_niagara_3) ENTRY(xor_niagara_4) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */ save %sp, -192, %sp @@ -536,6 +544,7 @@ ENTRY(xor_niagara_4) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */ ret restore ENDPROC(xor_niagara_4) +EXPORT_SYMBOL(xor_niagara_4) ENTRY(xor_niagara_5) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=src4 */ save %sp, -192, %sp @@ -634,3 +643,4 @@ ENTRY(xor_niagara_5) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=s ret restore ENDPROC(xor_niagara_5) +EXPORT_SYMBOL(xor_niagara_5) diff --git a/arch/sparc/math-emu/Makefile b/arch/sparc/math-emu/Makefile index 825dbee94d84..aea80597929b 100644 --- a/arch/sparc/math-emu/Makefile +++ b/arch/sparc/math-emu/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only # # Makefile for the FPU instruction emulation. # diff --git a/arch/sparc/math-emu/math_32.c b/arch/sparc/math-emu/math_32.c index aa4d55b0bdf0..d5beec856146 100644 --- a/arch/sparc/math-emu/math_32.c +++ b/arch/sparc/math-emu/math_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * arch/sparc/math-emu/math.c * @@ -68,7 +69,7 @@ #include <linux/sched.h> #include <linux/mm.h> #include <linux/perf_event.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include "sfp-util_32.h" #include <math-emu/soft-fp.h> @@ -358,7 +359,7 @@ static int do_one_mathemu(u32 insn, unsigned long *pfsr, unsigned long *fregs) *pfsr |= (6 << 14); return 0; /* simulate invalid_fp_register exception */ } - /* fall through */ + fallthrough; case 2: if (freg & 1) { /* doublewords must have bit 5 zeroed */ *pfsr |= (6 << 14); @@ -379,7 +380,7 @@ static int do_one_mathemu(u32 insn, unsigned long *pfsr, unsigned long *fregs) *pfsr |= (6 << 14); return 0; /* simulate invalid_fp_register exception */ } - /* fall through */ + fallthrough; case 2: if (freg & 1) { /* doublewords must have bit 5 zeroed */ *pfsr |= (6 << 14); @@ -407,13 +408,13 @@ static int do_one_mathemu(u32 insn, unsigned long *pfsr, unsigned long *fregs) *pfsr |= (6 << 14); return 0; /* simulate invalid_fp_register exception */ } - /* fall through */ + fallthrough; case 2: if (freg & 1) { /* doublewords must have bit 5 zeroed */ *pfsr |= (6 << 14); return 0; } - /* fall through */ + fallthrough; case 1: rd = (void *)&fregs[freg]; break; @@ -499,7 +500,7 @@ static int do_one_mathemu(u32 insn, unsigned long *pfsr, unsigned long *fregs) case 0: fsr = *pfsr; if (IR == -1) IR = 2; /* fcc is always fcc0 */ - fsr &= ~0xc00; fsr |= (IR << 10); break; + fsr &= ~0xc00; fsr |= (IR << 10); *pfsr = fsr; break; case 1: rd->s = IR; break; diff --git a/arch/sparc/math-emu/math_64.c b/arch/sparc/math-emu/math_64.c index 034aadbff036..1379dee26a65 100644 --- a/arch/sparc/math-emu/math_64.c +++ b/arch/sparc/math-emu/math_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * arch/sparc64/math-emu/math.c * @@ -15,7 +16,7 @@ #include <asm/fpumacro.h> #include <asm/ptrace.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/cacheflush.h> #include "sfp-util_64.h" diff --git a/arch/sparc/math-emu/sfp-util_32.h b/arch/sparc/math-emu/sfp-util_32.h index d1b2aff3c259..b57375ff28ee 100644 --- a/arch/sparc/math-emu/sfp-util_32.h +++ b/arch/sparc/math-emu/sfp-util_32.h @@ -1,23 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/kernel.h> #include <linux/sched.h> #include <linux/types.h> #include <asm/byteorder.h> #define add_ssaaaa(sh, sl, ah, al, bh, bl) \ - __asm__ ("addcc %r4,%5,%1\n\t" \ + __asm__ ("addcc %r4,%5,%1\n\t" \ "addx %r2,%3,%0\n" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "%rJ" ((USItype)(ah)), \ "rI" ((USItype)(bh)), \ "%rJ" ((USItype)(al)), \ "rI" ((USItype)(bl)) \ : "cc") #define sub_ddmmss(sh, sl, ah, al, bh, bl) \ - __asm__ ("subcc %r4,%5,%1\n\t" \ + __asm__ ("subcc %r4,%5,%1\n\t" \ "subx %r2,%3,%0\n" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "rJ" ((USItype)(ah)), \ "rI" ((USItype)(bh)), \ "rJ" ((USItype)(al)), \ @@ -65,8 +66,8 @@ "mulscc %%g1,0,%%g1\n\t" \ "add %%g1,%%g2,%0\n\t" \ "rd %%y,%1\n" \ - : "=r" ((USItype)(w1)), \ - "=r" ((USItype)(w0)) \ + : "=r" (w1), \ + "=r" (w0) \ : "%rI" ((USItype)(u)), \ "r" ((USItype)(v)) \ : "%g1", "%g2", "cc") @@ -98,8 +99,8 @@ "sub %1,%2,%1\n\t" \ "3: xnor %0,0,%0\n\t" \ "! End of inline udiv_qrnnd\n" \ - : "=&r" ((USItype)(q)), \ - "=&r" ((USItype)(r)) \ + : "=&r" (q), \ + "=&r" (r) \ : "r" ((USItype)(d)), \ "1" ((USItype)(n1)), \ "0" ((USItype)(n0)) : "%g1", "cc") diff --git a/arch/sparc/math-emu/sfp-util_64.h b/arch/sparc/math-emu/sfp-util_64.h index 425d3cf01af4..8fdb55aae9c8 100644 --- a/arch/sparc/math-emu/sfp-util_64.h +++ b/arch/sparc/math-emu/sfp-util_64.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * arch/sparc64/math-emu/sfp-util.h * @@ -17,8 +18,8 @@ "bcs,a,pn %%xcc, 1f\n\t" \ "add %0, 1, %0\n" \ "1:" \ - : "=r" ((UDItype)(sh)), \ - "=&r" ((UDItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "r" ((UDItype)(ah)), \ "r" ((UDItype)(bh)), \ "r" ((UDItype)(al)), \ @@ -31,8 +32,8 @@ "bcs,a,pn %%xcc, 1f\n\t" \ "sub %0, 1, %0\n" \ "1:" \ - : "=r" ((UDItype)(sh)), \ - "=&r" ((UDItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "r" ((UDItype)(ah)), \ "r" ((UDItype)(bh)), \ "r" ((UDItype)(al)), \ @@ -64,8 +65,8 @@ "sllx %3,32,%3\n\t" \ "add %1,%3,%1\n\t" \ "add %5,%2,%0" \ - : "=r" ((UDItype)(wh)), \ - "=&r" ((UDItype)(wl)), \ + : "=r" (wh), \ + "=&r" (wl), \ "=&r" (tmp1), "=&r" (tmp2), "=&r" (tmp3), "=&r" (tmp4) \ : "r" ((UDItype)(u)), \ "r" ((UDItype)(v)) \ diff --git a/arch/sparc/mm/Makefile b/arch/sparc/mm/Makefile index 30c3eccfdf5a..e9d232561c82 100644 --- a/arch/sparc/mm/Makefile +++ b/arch/sparc/mm/Makefile @@ -1,13 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 # Makefile for the linux Sparc-specific parts of the memory manager. # -asflags-y := -ansi -ccflags-y := -Werror - -obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o gup.o +obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o obj-y += fault_$(BITS).o obj-y += init_$(BITS).o -obj-$(CONFIG_SPARC32) += extable.o srmmu.o iommu.o io-unit.o +obj-$(CONFIG_SPARC32) += srmmu.o iommu.o io-unit.o obj-$(CONFIG_SPARC32) += srmmu_access.o obj-$(CONFIG_SPARC32) += hypersparc.o viking.o tsunami.o swift.o obj-$(CONFIG_SPARC32) += leon_mm.o @@ -15,5 +13,4 @@ obj-$(CONFIG_SPARC32) += leon_mm.o # Only used by sparc64 obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o -# Only used by sparc32 -obj-$(CONFIG_HIGHMEM) += highmem.o +obj-$(CONFIG_EXECMEM) += execmem.o diff --git a/arch/sparc/mm/execmem.c b/arch/sparc/mm/execmem.c new file mode 100644 index 000000000000..0fac97dd5728 --- /dev/null +++ b/arch/sparc/mm/execmem.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/mm.h> +#include <linux/execmem.h> + +static struct execmem_info execmem_info __ro_after_init; + +struct execmem_info __init *execmem_arch_setup(void) +{ + execmem_info = (struct execmem_info){ + .ranges = { + [EXECMEM_DEFAULT] = { + .start = MODULES_VADDR, + .end = MODULES_END, + .pgprot = PAGE_KERNEL, + .alignment = 1, + }, + }, + }; + + return &execmem_info; +} diff --git a/arch/sparc/mm/extable.c b/arch/sparc/mm/extable.c deleted file mode 100644 index a61c349448e1..000000000000 --- a/arch/sparc/mm/extable.c +++ /dev/null @@ -1,105 +0,0 @@ -/* - * linux/arch/sparc/mm/extable.c - */ - -#include <linux/module.h> -#include <asm/uaccess.h> - -void sort_extable(struct exception_table_entry *start, - struct exception_table_entry *finish) -{ -} - -/* Caller knows they are in a range if ret->fixup == 0 */ -const struct exception_table_entry * -search_extable(const struct exception_table_entry *start, - const struct exception_table_entry *last, - unsigned long value) -{ - const struct exception_table_entry *walk; - - /* Single insn entries are encoded as: - * word 1: insn address - * word 2: fixup code address - * - * Range entries are encoded as: - * word 1: first insn address - * word 2: 0 - * word 3: last insn address + 4 bytes - * word 4: fixup code address - * - * Deleted entries are encoded as: - * word 1: unused - * word 2: -1 - * - * See asm/uaccess.h for more details. - */ - - /* 1. Try to find an exact match. */ - for (walk = start; walk <= last; walk++) { - if (walk->fixup == 0) { - /* A range entry, skip both parts. */ - walk++; - continue; - } - - /* A deleted entry; see trim_init_extable */ - if (walk->fixup == -1) - continue; - - if (walk->insn == value) - return walk; - } - - /* 2. Try to find a range match. */ - for (walk = start; walk <= (last - 1); walk++) { - if (walk->fixup) - continue; - - if (walk[0].insn <= value && walk[1].insn > value) - return walk; - - walk++; - } - - return NULL; -} - -#ifdef CONFIG_MODULES -/* We could memmove them around; easier to mark the trimmed ones. */ -void trim_init_extable(struct module *m) -{ - unsigned int i; - bool range; - - for (i = 0; i < m->num_exentries; i += range ? 2 : 1) { - range = m->extable[i].fixup == 0; - - if (within_module_init(m->extable[i].insn, m)) { - m->extable[i].fixup = -1; - if (range) - m->extable[i+1].fixup = -1; - } - if (range) - i++; - } -} -#endif /* CONFIG_MODULES */ - -/* Special extable search, which handles ranges. Returns fixup */ -unsigned long search_extables_range(unsigned long addr, unsigned long *g2) -{ - const struct exception_table_entry *entry; - - entry = search_exception_tables(addr); - if (!entry) - return 0; - - /* Inside range? Fix g2 and return correct fixup */ - if (!entry->fixup) { - *g2 = (addr - entry->insn) / 4; - return (entry + 1)->fixup; - } - - return entry->fixup; -} diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index e98bfda205a2..86a831ebd8c8 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * fault.c: Page fault handlers for the Sparc. * @@ -21,19 +22,19 @@ #include <linux/perf_event.h> #include <linux/interrupt.h> #include <linux/kdebug.h> +#include <linux/uaccess.h> +#include <linux/extable.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/openprom.h> #include <asm/oplib.h> +#include <asm/setup.h> #include <asm/smp.h> #include <asm/traps.h> -#include <asm/uaccess.h> -int show_unhandled_signals = 1; +#include "mm_32.h" -static void unhandled_fault(unsigned long, struct task_struct *, - struct pt_regs *) __attribute__ ((noreturn)); +int show_unhandled_signals = 1; static void __noreturn unhandled_fault(unsigned long address, struct task_struct *tsk, @@ -54,54 +55,6 @@ static void __noreturn unhandled_fault(unsigned long address, die_if_kernel("Oops", regs); } -asmlinkage int lookup_fault(unsigned long pc, unsigned long ret_pc, - unsigned long address) -{ - struct pt_regs regs; - unsigned long g2; - unsigned int insn; - int i; - - i = search_extables_range(ret_pc, &g2); - switch (i) { - case 3: - /* load & store will be handled by fixup */ - return 3; - - case 1: - /* store will be handled by fixup, load will bump out */ - /* for _to_ macros */ - insn = *((unsigned int *) pc); - if ((insn >> 21) & 1) - return 1; - break; - - case 2: - /* load will be handled by fixup, store will bump out */ - /* for _from_ macros */ - insn = *((unsigned int *) pc); - if (!((insn >> 21) & 1) || ((insn>>19)&0x3f) == 15) - return 2; - break; - - default: - break; - } - - memset(®s, 0, sizeof(regs)); - regs.pc = pc; - regs.npc = pc + 4; - __asm__ __volatile__( - "rd %%psr, %0\n\t" - "nop\n\t" - "nop\n\t" - "nop\n" : "=r" (regs.psr)); - unhandled_fault(address, current, ®s); - - /* Not reached */ - return 0; -} - static inline void show_signal_msg(struct pt_regs *regs, int sig, int code, unsigned long address, struct task_struct *tsk) @@ -112,7 +65,7 @@ show_signal_msg(struct pt_regs *regs, int sig, int code, if (!printk_ratelimit()) return; - printk("%s%s[%d]: segfault at %lx ip %p (rpc %p) sp %p error %x", + printk("%s%s[%d]: segfault at %lx ip %px (rpc %px) sp %px error %x", task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG, tsk->comm, task_pid_nr(tsk), address, (void *)regs->pc, (void *)regs->u_regs[UREG_I7], @@ -126,24 +79,13 @@ show_signal_msg(struct pt_regs *regs, int sig, int code, static void __do_fault_siginfo(int code, int sig, struct pt_regs *regs, unsigned long addr) { - siginfo_t info; - - info.si_signo = sig; - info.si_code = code; - info.si_errno = 0; - info.si_addr = (void __user *) addr; - info.si_trapno = 0; - if (unlikely(show_unhandled_signals)) - show_signal_msg(regs, sig, info.si_code, + show_signal_msg(regs, sig, code, addr, current); - force_sig_info (sig, &info, current); + force_sig_fault(sig, code, (void __user *) addr); } -extern unsigned long safe_compute_effective_address(struct pt_regs *, - unsigned int); - static unsigned long compute_si_addr(struct pt_regs *regs, int text_fault) { unsigned int insn; @@ -173,12 +115,10 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, struct vm_area_struct *vma; struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; - unsigned int fixup; - unsigned long g2; int from_user = !(regs->psr & PSR_PS); - int fault, code; - unsigned int flags = (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE | - (write ? FAULT_FLAG_WRITE : 0)); + int code; + vm_fault_t fault; + unsigned int flags = FAULT_FLAG_DEFAULT; if (text_fault) address = regs->pc; @@ -200,31 +140,22 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm) + if (pagefault_disabled() || !mm) + goto no_context; + + if (!from_user && address >= PAGE_OFFSET) goto no_context; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); retry: - down_read(&mm->mmap_sem); - - if (!from_user && address >= PAGE_OFFSET) - goto bad_area; - - vma = find_vma(mm, address); + vma = lock_mm_and_find_vma(mm, address, regs); if (!vma) - goto bad_area; - if (vma->vm_start <= address) - goto good_area; - if (!(vma->vm_flags & VM_GROWSDOWN)) - goto bad_area; - if (expand_stack(vma, address)) - goto bad_area; + goto bad_area_nosemaphore; /* * Ok, we have a good vm_area for this memory access, so * we can handle it.. */ -good_area: code = SEGV_ACCERR; if (write) { if (!(vma->vm_flags & VM_WRITE)) @@ -235,48 +166,50 @@ good_area: goto bad_area; } + if (from_user) + flags |= FAULT_FLAG_USER; + if (write) + flags |= FAULT_FLAG_WRITE; + /* * If for any reason at all we couldn't handle the fault, * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags, regs); + + if (fault_signal_pending(fault, regs)) { + if (!from_user) + goto no_context; + return; + } - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) return; if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; + else if (fault & VM_FAULT_SIGSEGV) + goto bad_area; else if (fault & VM_FAULT_SIGBUS) goto do_sigbus; BUG(); } - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_MAJOR) { - current->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, - 1, regs, address); - } else { - current->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, - 1, regs, address); - } - if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; - flags |= FAULT_FLAG_TRIED; + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; - /* No need to up_read(&mm->mmap_sem) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ + /* No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ - goto retry; - } + goto retry; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; /* @@ -284,7 +217,7 @@ good_area: * Fix it, but check if it's kernel or user first.. */ bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); bad_area_nosemaphore: /* User mode accesses just cause a SIGSEGV */ @@ -295,45 +228,29 @@ bad_area_nosemaphore: /* Is this in ex_table? */ no_context: - g2 = regs->u_regs[UREG_G2]; if (!from_user) { - fixup = search_extables_range(regs->pc, &g2); - /* Values below 10 are reserved for other things */ - if (fixup > 10) { - extern const unsigned __memset_start[]; - extern const unsigned __memset_end[]; - extern const unsigned __csum_partial_copy_start[]; - extern const unsigned __csum_partial_copy_end[]; + const struct exception_table_entry *entry; + entry = search_exception_tables(regs->pc); #ifdef DEBUG_EXCEPTIONS - printk("Exception: PC<%08lx> faddr<%08lx>\n", - regs->pc, address); - printk("EX_TABLE: insn<%08lx> fixup<%08x> g2<%08lx>\n", - regs->pc, fixup, g2); + printk("Exception: PC<%08lx> faddr<%08lx>\n", + regs->pc, address); + printk("EX_TABLE: insn<%08lx> fixup<%08x>\n", + regs->pc, entry->fixup); #endif - if ((regs->pc >= (unsigned long)__memset_start && - regs->pc < (unsigned long)__memset_end) || - (regs->pc >= (unsigned long)__csum_partial_copy_start && - regs->pc < (unsigned long)__csum_partial_copy_end)) { - regs->u_regs[UREG_I4] = address; - regs->u_regs[UREG_I5] = regs->pc; - } - regs->u_regs[UREG_G2] = g2; - regs->pc = fixup; - regs->npc = regs->pc + 4; - return; - } + regs->pc = entry->fixup; + regs->npc = regs->pc + 4; + return; } unhandled_fault(address, tsk, regs); - do_exit(SIGKILL); /* * We ran out of memory, or some other thing happened to us that made * us unable to handle the page fault gracefully. */ out_of_memory: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (from_user) { pagefault_out_of_memory(); return; @@ -341,7 +258,7 @@ out_of_memory: goto no_context; do_sigbus: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, text_fault); if (!from_user) goto no_context; @@ -354,6 +271,8 @@ vmalloc_fault: */ int offset = pgd_index(address); pgd_t *pgd, *pgd_k; + p4d_t *p4d, *p4d_k; + pud_t *pud, *pud_k; pmd_t *pmd, *pmd_k; pgd = tsk->active_mm->pgd + offset; @@ -366,8 +285,13 @@ vmalloc_fault: return; } - pmd = pmd_offset(pgd, address); - pmd_k = pmd_offset(pgd_k, address); + p4d = p4d_offset(pgd, address); + pud = pud_offset(p4d, address); + pmd = pmd_offset(pud, address); + + p4d_k = p4d_offset(pgd_k, address); + pud_k = pud_offset(p4d_k, address); + pmd_k = pmd_offset(pud_k, address); if (pmd_present(*pmd) || !pmd_present(*pmd_k)) goto bad_area_nosemaphore; @@ -383,50 +307,45 @@ static void force_user_fault(unsigned long address, int write) struct vm_area_struct *vma; struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; + unsigned int flags = FAULT_FLAG_USER; int code; code = SEGV_MAPERR; - down_read(&mm->mmap_sem); - vma = find_vma(mm, address); + vma = lock_mm_and_find_vma(mm, address, NULL); if (!vma) - goto bad_area; - if (vma->vm_start <= address) - goto good_area; - if (!(vma->vm_flags & VM_GROWSDOWN)) - goto bad_area; - if (expand_stack(vma, address)) - goto bad_area; -good_area: + goto bad_area_nosemaphore; code = SEGV_ACCERR; if (write) { if (!(vma->vm_flags & VM_WRITE)) goto bad_area; + flags |= FAULT_FLAG_WRITE; } else { if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } - switch (handle_mm_fault(mm, vma, address, write ? FAULT_FLAG_WRITE : 0)) { + switch (handle_mm_fault(vma, address, flags, NULL)) { case VM_FAULT_SIGBUS: case VM_FAULT_OOM: goto do_sigbus; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); return; bad_area: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); +bad_area_nosemaphore: __do_fault_siginfo(code, SIGSEGV, tsk->thread.kregs, address); return; do_sigbus: - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); __do_fault_siginfo(BUS_ADRERR, SIGBUS, tsk->thread.kregs, address); } static void check_stack_aligned(unsigned long sp) { if (sp & 0x7UL) - force_sig(SIGILL, current); + force_sig(SIGILL); } void window_overflow_fault(void) diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index 5062ff389e83..e326caf708c6 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * arch/sparc64/mm/fault.c: Page fault handlers for the 64-bit Sparc. * @@ -10,44 +11,32 @@ #include <linux/string.h> #include <linux/types.h> #include <linux/sched.h> +#include <linux/sched/debug.h> #include <linux/ptrace.h> #include <linux/mman.h> #include <linux/signal.h> #include <linux/mm.h> -#include <linux/module.h> +#include <linux/extable.h> #include <linux/init.h> #include <linux/perf_event.h> #include <linux/interrupt.h> #include <linux/kprobes.h> #include <linux/kdebug.h> #include <linux/percpu.h> +#include <linux/context_tracking.h> +#include <linux/uaccess.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/openprom.h> #include <asm/oplib.h> -#include <asm/uaccess.h> #include <asm/asi.h> #include <asm/lsu.h> #include <asm/sections.h> #include <asm/mmu_context.h> +#include <asm/setup.h> int show_unhandled_signals = 1; -static inline __kprobes int notify_page_fault(struct pt_regs *regs) -{ - int ret = 0; - - /* kprobe_running() needs smp_processor_id() */ - if (kprobes_built_in() && !user_mode(regs)) { - preempt_disable(); - if (kprobe_running() && kprobe_fault_handler(regs, 0)) - ret = 1; - preempt_enable(); - } - return ret; -} - static void __kprobes unhandled_fault(unsigned long address, struct task_struct *tsk, struct pt_regs *regs) @@ -81,7 +70,7 @@ static void __kprobes bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr) } /* - * We now make sure that mmap_sem is held in all paths that call + * We now make sure that mmap_lock is held in all paths that call * this. Additionally, to prevent kswapd from ripping ptes from * under us, raise interrupts around the time that we look at the * pte, kswapd will have to wait to get his smp ipi response from @@ -90,43 +79,60 @@ static void __kprobes bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr) static unsigned int get_user_insn(unsigned long tpc) { pgd_t *pgdp = pgd_offset(current->mm, tpc); + p4d_t *p4dp; pud_t *pudp; pmd_t *pmdp; pte_t *ptep, pte; unsigned long pa; u32 insn = 0; - unsigned long pstate; - if (pgd_none(*pgdp)) - goto outret; - pudp = pud_offset(pgdp, tpc); - if (pud_none(*pudp)) - goto outret; - pmdp = pmd_offset(pudp, tpc); - if (pmd_none(*pmdp)) - goto outret; - - /* This disables preemption for us as well. */ - __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate)); - __asm__ __volatile__("wrpr %0, %1, %%pstate" - : : "r" (pstate), "i" (PSTATE_IE)); - ptep = pte_offset_map(pmdp, tpc); - pte = *ptep; - if (!pte_present(pte)) + if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) + goto out; + p4dp = p4d_offset(pgdp, tpc); + if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp))) + goto out; + pudp = pud_offset(p4dp, tpc); + if (pud_none(*pudp) || unlikely(pud_bad(*pudp))) goto out; - pa = (pte_pfn(pte) << PAGE_SHIFT); - pa += (tpc & ~PAGE_MASK); + /* This disables preemption for us as well. */ + local_irq_disable(); - /* Use phys bypass so we don't pollute dtlb/dcache. */ - __asm__ __volatile__("lduwa [%1] %2, %0" - : "=r" (insn) - : "r" (pa), "i" (ASI_PHYS_USE_EC)); + pmdp = pmd_offset(pudp, tpc); +again: + if (pmd_none(*pmdp) || unlikely(pmd_bad(*pmdp))) + goto out_irq_enable; +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + if (is_hugetlb_pmd(*pmdp)) { + pa = pmd_pfn(*pmdp) << PAGE_SHIFT; + pa += tpc & ~HPAGE_MASK; + + /* Use phys bypass so we don't pollute dtlb/dcache. */ + __asm__ __volatile__("lduwa [%1] %2, %0" + : "=r" (insn) + : "r" (pa), "i" (ASI_PHYS_USE_EC)); + } else +#endif + { + ptep = pte_offset_map(pmdp, tpc); + if (!ptep) + goto again; + pte = *ptep; + if (pte_present(pte)) { + pa = (pte_pfn(pte) << PAGE_SHIFT); + pa += (tpc & ~PAGE_MASK); + + /* Use phys bypass so we don't pollute dtlb/dcache. */ + __asm__ __volatile__("lduwa [%1] %2, %0" + : "=r" (insn) + : "r" (pa), "i" (ASI_PHYS_USE_EC)); + } + pte_unmap(ptep); + } +out_irq_enable: + local_irq_enable(); out: - pte_unmap(ptep); - __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate)); -outret: return insn; } @@ -140,7 +146,7 @@ show_signal_msg(struct pt_regs *regs, int sig, int code, if (!printk_ratelimit()) return; - printk("%s%s[%d]: segfault at %lx ip %p (rpc %p) sp %p error %x", + printk("%s%s[%d]: segfault at %lx ip %px (rpc %px) sp %px error %x", task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG, tsk->comm, task_pid_nr(tsk), address, (void *)regs->tpc, (void *)regs->u_regs[UREG_I7], @@ -152,30 +158,30 @@ show_signal_msg(struct pt_regs *regs, int sig, int code, } static void do_fault_siginfo(int code, int sig, struct pt_regs *regs, - unsigned int insn, int fault_code) + unsigned long fault_addr, unsigned int insn, + int fault_code) { unsigned long addr; - siginfo_t info; - info.si_code = code; - info.si_signo = sig; - info.si_errno = 0; - if (fault_code & FAULT_CODE_ITLB) + if (fault_code & FAULT_CODE_ITLB) { addr = regs->tpc; - else - addr = compute_effective_address(regs, insn, 0); - info.si_addr = (void __user *) addr; - info.si_trapno = 0; + } else { + /* If we were able to probe the faulting instruction, use it + * to compute a precise fault address. Otherwise use the fault + * time provided address which may only have page granularity. + */ + if (insn) + addr = compute_effective_address(regs, insn, 0); + else + addr = fault_addr; + } if (unlikely(show_unhandled_signals)) show_signal_msg(regs, sig, code, addr, current); - force_sig_info(sig, &info, current); + force_sig_fault(sig, code, (void __user *) addr); } -extern int handle_ldf_stq(u32, struct pt_regs *); -extern int handle_ld_nf(u32, struct pt_regs *); - static unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn) { if (!insn) { @@ -238,7 +244,7 @@ static void __kprobes do_kernel_fault(struct pt_regs *regs, int si_code, /* The si_code was set to make clear whether * this was a SEGV_MAPERR or SEGV_ACCERR fault. */ - do_fault_siginfo(si_code, SIGSEGV, regs, insn, fault_code); + do_fault_siginfo(si_code, SIGSEGV, regs, address, insn, fault_code); return; } @@ -258,31 +264,21 @@ static void noinline __kprobes bogus_32bit_fault_tpc(struct pt_regs *regs) show_regs(regs); } -static void noinline __kprobes bogus_32bit_fault_address(struct pt_regs *regs, - unsigned long addr) -{ - static int times; - - if (times++ < 10) - printk(KERN_ERR "FAULT[%s:%d]: 32-bit process " - "reports 64-bit fault address [%lx]\n", - current->comm, current->pid, addr); - show_regs(regs); -} - asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); struct mm_struct *mm = current->mm; struct vm_area_struct *vma; unsigned int insn = 0; - int si_code, fault_code, fault; + int si_code, fault_code; + vm_fault_t fault; unsigned long address, mm_rss; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; fault_code = get_thread_fault_code(); - if (notify_page_fault(regs)) - return; + if (kprobe_page_fault(regs, 0)) + goto exit_exception; si_code = SEGV_MAPERR; address = current_thread_info()->fault_address; @@ -298,10 +294,8 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) goto intr_or_no_mm; } } - if (unlikely((address >> 32) != 0)) { - bogus_32bit_fault_address(regs, address); + if (unlikely((address >> 32) != 0)) goto intr_or_no_mm; - } } if (regs->tstate & TSTATE_PRIV) { @@ -313,20 +307,21 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) /* Valid, no problems... */ } else { bad_kernel_pc(regs, address); - return; + goto exit_exception; } - } + } else + flags |= FAULT_FLAG_USER; /* * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm) + if (faulthandler_disabled() || !mm) goto intr_or_no_mm; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); - if (!down_read_trylock(&mm->mmap_sem)) { + if (!mmap_read_trylock(mm)) { if ((regs->tstate & TSTATE_PRIV) && !search_exception_tables(regs->tpc)) { insn = get_fault_insn(regs, insn); @@ -334,9 +329,12 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) } retry: - down_read(&mm->mmap_sem); + mmap_read_lock(mm); } + if (fault_code & FAULT_CODE_BAD_RA) + goto do_sigbus; + vma = find_vma(mm, address); if (!vma) goto bad_area; @@ -388,8 +386,9 @@ continue_fault: goto bad_area; } } - if (expand_stack(vma, address)) - goto bad_area; + vma = expand_stack(mm, address); + if (!vma) + goto bad_area_nosemaphore; /* * Ok, we have a good vm_area for this memory access, so * we can handle it.. @@ -401,8 +400,9 @@ good_area: * that here. */ if ((fault_code & FAULT_CODE_ITLB) && !(vma->vm_flags & VM_EXEC)) { - BUG_ON(address != regs->tpc); - BUG_ON(regs->tstate & TSTATE_PRIV); + WARN(address != regs->tpc, + "address (%lx) != regs->tpc (%lx)\n", address, regs->tpc); + WARN_ON(regs->tstate & TSTATE_PRIV); goto bad_area; } @@ -418,59 +418,61 @@ good_area: vma->vm_file != NULL) set_thread_fault_code(fault_code | FAULT_CODE_BLKCOMMIT); + + flags |= FAULT_FLAG_WRITE; } else { /* Allow reads even for write-only mappings */ if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } - flags |= ((fault_code & FAULT_CODE_WRITE) ? FAULT_FLAG_WRITE : 0); - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags, regs); - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) - return; + if (fault_signal_pending(fault, regs)) { + if (regs->tstate & TSTATE_PRIV) { + insn = get_fault_insn(regs, insn); + goto handle_kernel_fault; + } + goto exit_exception; + } + + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + goto lock_released; if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; + else if (fault & VM_FAULT_SIGSEGV) + goto bad_area; else if (fault & VM_FAULT_SIGBUS) goto do_sigbus; BUG(); } - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_MAJOR) { - current->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, - 1, regs, address); - } else { - current->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, - 1, regs, address); - } - if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; - flags |= FAULT_FLAG_TRIED; + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; - /* No need to up_read(&mm->mmap_sem) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ + /* No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ - goto retry; - } + goto retry; } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); +lock_released: mm_rss = get_mm_rss(mm); -#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE)); +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) + mm_rss -= (mm->context.thp_pte_count * (HPAGE_SIZE / PAGE_SIZE)); #endif if (unlikely(mm_rss > mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit)) tsb_grow(mm, MM_TSB_BASE, mm_rss); #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - mm_rss = mm->context.huge_pte_count; + mm_rss = mm->context.hugetlb_pte_count + mm->context.thp_pte_count; + mm_rss *= REAL_HPAGE_PER_HPAGE; if (unlikely(mm_rss > mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) { if (mm->context.tsb_block[MM_TSB_HUGE].tsb) @@ -480,6 +482,8 @@ good_area: } #endif +exit_exception: + exception_exit(prev_state); return; /* @@ -487,12 +491,13 @@ good_area: * Fix it, but check if it's kernel or user first.. */ bad_area: + mmap_read_unlock(mm); +bad_area_nosemaphore: insn = get_fault_insn(regs, insn); - up_read(&mm->mmap_sem); handle_kernel_fault: do_kernel_fault(regs, si_code, fault_code, insn, address); - return; + goto exit_exception; /* * We ran out of memory, or some other thing happened to us that made @@ -500,10 +505,10 @@ handle_kernel_fault: */ out_of_memory: insn = get_fault_insn(regs, insn); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); if (!(regs->tstate & TSTATE_PRIV)) { pagefault_out_of_memory(); - return; + goto exit_exception; } goto handle_kernel_fault; @@ -513,13 +518,13 @@ intr_or_no_mm: do_sigbus: insn = get_fault_insn(regs, insn); - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); /* * Send a sigbus, regardless of whether we were in kernel * or user mode. */ - do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, insn, fault_code); + do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, address, insn, fault_code); /* Kernel mode? Handle exceptions or die */ if (regs->tstate & TSTATE_PRIV) diff --git a/arch/sparc/mm/gup.c b/arch/sparc/mm/gup.c deleted file mode 100644 index 01ee23dd724d..000000000000 --- a/arch/sparc/mm/gup.c +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Lockless get_user_pages_fast for sparc, cribbed from powerpc - * - * Copyright (C) 2008 Nick Piggin - * Copyright (C) 2008 Novell Inc. - */ - -#include <linux/sched.h> -#include <linux/mm.h> -#include <linux/vmstat.h> -#include <linux/pagemap.h> -#include <linux/rwsem.h> -#include <asm/pgtable.h> - -/* - * The performance critical leaf functions are made noinline otherwise gcc - * inlines everything into a single function which results in too much - * register pressure. - */ -static noinline int gup_pte_range(pmd_t pmd, unsigned long addr, - unsigned long end, int write, struct page **pages, int *nr) -{ - unsigned long mask, result; - pte_t *ptep; - - if (tlb_type == hypervisor) { - result = _PAGE_PRESENT_4V|_PAGE_P_4V; - if (write) - result |= _PAGE_WRITE_4V; - } else { - result = _PAGE_PRESENT_4U|_PAGE_P_4U; - if (write) - result |= _PAGE_WRITE_4U; - } - mask = result | _PAGE_SPECIAL; - - ptep = pte_offset_kernel(&pmd, addr); - do { - struct page *page, *head; - pte_t pte = *ptep; - - if ((pte_val(pte) & mask) != result) - return 0; - VM_BUG_ON(!pfn_valid(pte_pfn(pte))); - - /* The hugepage case is simplified on sparc64 because - * we encode the sub-page pfn offsets into the - * hugepage PTEs. We could optimize this in the future - * use page_cache_add_speculative() for the hugepage case. - */ - page = pte_page(pte); - head = compound_head(page); - if (!page_cache_get_speculative(head)) - return 0; - if (unlikely(pte_val(pte) != pte_val(*ptep))) { - put_page(head); - return 0; - } - if (head != page) - get_huge_page_tail(page); - - pages[*nr] = page; - (*nr)++; - } while (ptep++, addr += PAGE_SIZE, addr != end); - - return 1; -} - -static int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr, - unsigned long end, int write, struct page **pages, - int *nr) -{ - struct page *head, *page, *tail; - u32 mask; - int refs; - - mask = PMD_HUGE_PRESENT; - if (write) - mask |= PMD_HUGE_WRITE; - if ((pmd_val(pmd) & mask) != mask) - return 0; - - refs = 0; - head = pmd_page(pmd); - page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT); - tail = page; - do { - VM_BUG_ON(compound_head(page) != head); - pages[*nr] = page; - (*nr)++; - page++; - refs++; - } while (addr += PAGE_SIZE, addr != end); - - if (!page_cache_add_speculative(head, refs)) { - *nr -= refs; - return 0; - } - - if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) { - *nr -= refs; - while (refs--) - put_page(head); - return 0; - } - - /* Any tail page need their mapcount reference taken before we - * return. - */ - while (refs--) { - if (PageTail(tail)) - get_huge_page_tail(tail); - tail++; - } - - return 1; -} - -static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end, - int write, struct page **pages, int *nr) -{ - unsigned long next; - pmd_t *pmdp; - - pmdp = pmd_offset(&pud, addr); - do { - pmd_t pmd = *pmdp; - - next = pmd_addr_end(addr, end); - if (pmd_none(pmd) || pmd_trans_splitting(pmd)) - return 0; - if (unlikely(pmd_large(pmd))) { - if (!gup_huge_pmd(pmdp, pmd, addr, next, - write, pages, nr)) - return 0; - } else if (!gup_pte_range(pmd, addr, next, write, - pages, nr)) - return 0; - } while (pmdp++, addr = next, addr != end); - - return 1; -} - -static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end, - int write, struct page **pages, int *nr) -{ - unsigned long next; - pud_t *pudp; - - pudp = pud_offset(&pgd, addr); - do { - pud_t pud = *pudp; - - next = pud_addr_end(addr, end); - if (pud_none(pud)) - return 0; - if (!gup_pmd_range(pud, addr, next, write, pages, nr)) - return 0; - } while (pudp++, addr = next, addr != end); - - return 1; -} - -int get_user_pages_fast(unsigned long start, int nr_pages, int write, - struct page **pages) -{ - struct mm_struct *mm = current->mm; - unsigned long addr, len, end; - unsigned long next; - pgd_t *pgdp; - int nr = 0; - - start &= PAGE_MASK; - addr = start; - len = (unsigned long) nr_pages << PAGE_SHIFT; - end = start + len; - - /* - * XXX: batch / limit 'nr', to avoid large irq off latency - * needs some instrumenting to determine the common sizes used by - * important workloads (eg. DB2), and whether limiting the batch size - * will decrease performance. - * - * It seems like we're in the clear for the moment. Direct-IO is - * the main guy that batches up lots of get_user_pages, and even - * they are limited to 64-at-a-time which is not so many. - */ - /* - * This doesn't prevent pagetable teardown, but does prevent - * the pagetables from being freed on sparc. - * - * So long as we atomically load page table pointers versus teardown, - * we can follow the address down to the the page and take a ref on it. - */ - local_irq_disable(); - - pgdp = pgd_offset(mm, addr); - do { - pgd_t pgd = *pgdp; - - next = pgd_addr_end(addr, end); - if (pgd_none(pgd)) - goto slow; - if (!gup_pud_range(pgd, addr, next, write, pages, &nr)) - goto slow; - } while (pgdp++, addr = next, addr != end); - - local_irq_enable(); - - VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT); - return nr; - - { - int ret; - -slow: - local_irq_enable(); - - /* Try to get the remaining pages with get_user_pages */ - start += nr << PAGE_SHIFT; - pages += nr; - - down_read(&mm->mmap_sem); - ret = get_user_pages(current, mm, start, - (end - start) >> PAGE_SHIFT, write, 0, pages, NULL); - up_read(&mm->mmap_sem); - - /* Have to be a bit careful with return values */ - if (nr > 0) { - if (ret < 0) - ret = nr; - else - ret += nr; - } - - return ret; - } -} diff --git a/arch/sparc/mm/highmem.c b/arch/sparc/mm/highmem.c deleted file mode 100644 index 449f864f0cef..000000000000 --- a/arch/sparc/mm/highmem.c +++ /dev/null @@ -1,130 +0,0 @@ -/* - * highmem.c: virtual kernel memory mappings for high memory - * - * Provides kernel-static versions of atomic kmap functions originally - * found as inlines in include/asm-sparc/highmem.h. These became - * needed as kmap_atomic() and kunmap_atomic() started getting - * called from within modules. - * -- Tomas Szepe <szepe@pinerecords.com>, September 2002 - * - * But kmap_atomic() and kunmap_atomic() cannot be inlined in - * modules because they are loaded with btfixup-ped functions. - */ - -/* - * The use of kmap_atomic/kunmap_atomic is discouraged - kmap/kunmap - * gives a more generic (and caching) interface. But kmap_atomic can - * be used in IRQ contexts, so in some (very limited) cases we need it. - * - * XXX This is an old text. Actually, it's good to use atomic kmaps, - * provided you remember that they are atomic and not try to sleep - * with a kmap taken, much like a spinlock. Non-atomic kmaps are - * shared by CPUs, and so precious, and establishing them requires IPI. - * Atomic kmaps are lightweight and we may have NCPUS more of them. - */ -#include <linux/highmem.h> -#include <linux/export.h> -#include <linux/mm.h> - -#include <asm/cacheflush.h> -#include <asm/tlbflush.h> -#include <asm/pgalloc.h> -#include <asm/vaddrs.h> - -pgprot_t kmap_prot; - -static pte_t *kmap_pte; - -void __init kmap_init(void) -{ - unsigned long address; - pmd_t *dir; - - address = __fix_to_virt(FIX_KMAP_BEGIN); - dir = pmd_offset(pgd_offset_k(address), address); - - /* cache the first kmap pte */ - kmap_pte = pte_offset_kernel(dir, address); - kmap_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE); -} - -void *kmap_atomic(struct page *page) -{ - unsigned long vaddr; - long idx, type; - - /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ - pagefault_disable(); - if (!PageHighMem(page)) - return page_address(page); - - type = kmap_atomic_idx_push(); - idx = type + KM_TYPE_NR*smp_processor_id(); - vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); - -/* XXX Fix - Anton */ -#if 0 - __flush_cache_one(vaddr); -#else - flush_cache_all(); -#endif - -#ifdef CONFIG_DEBUG_HIGHMEM - BUG_ON(!pte_none(*(kmap_pte-idx))); -#endif - set_pte(kmap_pte-idx, mk_pte(page, kmap_prot)); -/* XXX Fix - Anton */ -#if 0 - __flush_tlb_one(vaddr); -#else - flush_tlb_all(); -#endif - - return (void*) vaddr; -} -EXPORT_SYMBOL(kmap_atomic); - -void __kunmap_atomic(void *kvaddr) -{ - unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; - int type; - - if (vaddr < FIXADDR_START) { // FIXME - pagefault_enable(); - return; - } - - type = kmap_atomic_idx(); - -#ifdef CONFIG_DEBUG_HIGHMEM - { - unsigned long idx; - - idx = type + KM_TYPE_NR * smp_processor_id(); - BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN+idx)); - - /* XXX Fix - Anton */ -#if 0 - __flush_cache_one(vaddr); -#else - flush_cache_all(); -#endif - - /* - * force other mappings to Oops if they'll try to access - * this pte without first remap it - */ - pte_clear(&init_mm, vaddr, kmap_pte-idx); - /* XXX Fix - Anton */ -#if 0 - __flush_tlb_one(vaddr); -#else - flush_tlb_all(); -#endif - } -#endif - - kmap_atomic_idx_pop(); - pagefault_enable(); -} -EXPORT_SYMBOL(__kunmap_atomic); diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index d2b59441ebdd..4652e868663b 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c @@ -1,12 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 /* * SPARC64 Huge TLB page support. * * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net) */ -#include <linux/init.h> #include <linux/fs.h> #include <linux/mm.h> +#include <linux/sched/mm.h> #include <linux/hugetlb.h> #include <linux/pagemap.h> #include <linux/sysctl.h> @@ -18,224 +19,299 @@ #include <asm/cacheflush.h> #include <asm/mmu_context.h> -/* Slightly simplified from the non-hugepage variant because by - * definition we don't have to worry about any page coloring stuff - */ -#define VA_EXCLUDE_START (0x0000080000000000UL - (1UL << 32UL)) -#define VA_EXCLUDE_END (0xfffff80000000000UL + (1UL << 32UL)) - -static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, - unsigned long addr, - unsigned long len, - unsigned long pgoff, - unsigned long flags) + +static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift) { - unsigned long task_size = TASK_SIZE; - struct vm_unmapped_area_info info; - - if (test_thread_flag(TIF_32BIT)) - task_size = STACK_TOP32; - - info.flags = 0; - info.length = len; - info.low_limit = TASK_UNMAPPED_BASE; - info.high_limit = min(task_size, VA_EXCLUDE_START); - info.align_mask = PAGE_MASK & ~HPAGE_MASK; - info.align_offset = 0; - addr = vm_unmapped_area(&info); - - if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) { - VM_BUG_ON(addr != -ENOMEM); - info.low_limit = VA_EXCLUDE_END; - info.high_limit = task_size; - addr = vm_unmapped_area(&info); + unsigned long hugepage_size = _PAGE_SZ4MB_4U; + + pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4U; + + switch (shift) { + case HPAGE_256MB_SHIFT: + hugepage_size = _PAGE_SZ256MB_4U; + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_SHIFT: + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_64K_SHIFT: + hugepage_size = _PAGE_SZ64K_4U; + break; + default: + WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift); } - return addr; + pte_val(entry) = pte_val(entry) | hugepage_size; + return entry; } -static unsigned long -hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, - const unsigned long len, - const unsigned long pgoff, - const unsigned long flags) +static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift) { - struct mm_struct *mm = current->mm; - unsigned long addr = addr0; - struct vm_unmapped_area_info info; - - /* This should only ever run for 32-bit processes. */ - BUG_ON(!test_thread_flag(TIF_32BIT)); - - info.flags = VM_UNMAPPED_AREA_TOPDOWN; - info.length = len; - info.low_limit = PAGE_SIZE; - info.high_limit = mm->mmap_base; - info.align_mask = PAGE_MASK & ~HPAGE_MASK; - info.align_offset = 0; - addr = vm_unmapped_area(&info); - - /* - * A failed mmap() very likely causes application failure, - * so fall back to the bottom-up function here. This scenario - * can happen with large stack limits and large mmap() - * allocations. - */ - if (addr & ~PAGE_MASK) { - VM_BUG_ON(addr != -ENOMEM); - info.flags = 0; - info.low_limit = TASK_UNMAPPED_BASE; - info.high_limit = STACK_TOP32; - addr = vm_unmapped_area(&info); + unsigned long hugepage_size = _PAGE_SZ4MB_4V; + + pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V; + + switch (shift) { + case HPAGE_16GB_SHIFT: + hugepage_size = _PAGE_SZ16GB_4V; + pte_val(entry) |= _PAGE_PUD_HUGE; + break; + case HPAGE_2GB_SHIFT: + hugepage_size = _PAGE_SZ2GB_4V; + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_256MB_SHIFT: + hugepage_size = _PAGE_SZ256MB_4V; + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_SHIFT: + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_64K_SHIFT: + hugepage_size = _PAGE_SZ64K_4V; + break; + default: + WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift); } - return addr; + pte_val(entry) = pte_val(entry) | hugepage_size; + return entry; +} + +static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int shift) +{ + if (tlb_type == hypervisor) + return sun4v_hugepage_shift_to_tte(entry, shift); + else + return sun4u_hugepage_shift_to_tte(entry, shift); +} + +pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags) +{ + pte_t pte; + + entry = pte_mkhuge(entry); + pte = hugepage_shift_to_tte(entry, shift); + +#ifdef CONFIG_SPARC64 + /* If this vma has ADI enabled on it, turn on TTE.mcd + */ + if (flags & VM_SPARC_ADI) + return pte_mkmcd(pte); + else + return pte_mknotmcd(pte); +#else + return pte; +#endif } -unsigned long -hugetlb_get_unmapped_area(struct file *file, unsigned long addr, - unsigned long len, unsigned long pgoff, unsigned long flags) +static unsigned int sun4v_huge_tte_to_shift(pte_t entry) { - struct mm_struct *mm = current->mm; - struct vm_area_struct *vma; - unsigned long task_size = TASK_SIZE; - - if (test_thread_flag(TIF_32BIT)) - task_size = STACK_TOP32; - - if (len & ~HPAGE_MASK) - return -EINVAL; - if (len > task_size) - return -ENOMEM; - - if (flags & MAP_FIXED) { - if (prepare_hugepage_range(file, addr, len)) - return -EINVAL; - return addr; + unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4V; + unsigned int shift; + + switch (tte_szbits) { + case _PAGE_SZ16GB_4V: + shift = HPAGE_16GB_SHIFT; + break; + case _PAGE_SZ2GB_4V: + shift = HPAGE_2GB_SHIFT; + break; + case _PAGE_SZ256MB_4V: + shift = HPAGE_256MB_SHIFT; + break; + case _PAGE_SZ4MB_4V: + shift = REAL_HPAGE_SHIFT; + break; + case _PAGE_SZ64K_4V: + shift = HPAGE_64K_SHIFT; + break; + default: + shift = PAGE_SHIFT; + break; } + return shift; +} - if (addr) { - addr = ALIGN(addr, HPAGE_SIZE); - vma = find_vma(mm, addr); - if (task_size - len >= addr && - (!vma || addr + len <= vma->vm_start)) - return addr; +static unsigned int sun4u_huge_tte_to_shift(pte_t entry) +{ + unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4U; + unsigned int shift; + + switch (tte_szbits) { + case _PAGE_SZ256MB_4U: + shift = HPAGE_256MB_SHIFT; + break; + case _PAGE_SZ4MB_4U: + shift = REAL_HPAGE_SHIFT; + break; + case _PAGE_SZ64K_4U: + shift = HPAGE_64K_SHIFT; + break; + default: + shift = PAGE_SHIFT; + break; } - if (mm->get_unmapped_area == arch_get_unmapped_area) - return hugetlb_get_unmapped_area_bottomup(file, addr, len, - pgoff, flags); - else - return hugetlb_get_unmapped_area_topdown(file, addr, len, - pgoff, flags); + return shift; } -pte_t *huge_pte_alloc(struct mm_struct *mm, +static unsigned long tte_to_shift(pte_t entry) +{ + if (tlb_type == hypervisor) + return sun4v_huge_tte_to_shift(entry); + + return sun4u_huge_tte_to_shift(entry); +} + +static unsigned int huge_tte_to_shift(pte_t entry) +{ + unsigned long shift = tte_to_shift(entry); + + if (shift == PAGE_SHIFT) + WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n", + pte_val(entry)); + + return shift; +} + +static unsigned long huge_tte_to_size(pte_t pte) +{ + unsigned long size = 1UL << huge_tte_to_shift(pte); + + if (size == REAL_HPAGE_SIZE) + size = HPAGE_SIZE; + return size; +} + +unsigned long pud_leaf_size(pud_t pud) { return 1UL << tte_to_shift(*(pte_t *)&pud); } +unsigned long pmd_leaf_size(pmd_t pmd) { return 1UL << tte_to_shift(*(pte_t *)&pmd); } +unsigned long pte_leaf_size(pte_t pte) { return 1UL << tte_to_shift(pte); } + +pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, unsigned long sz) { pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; - pte_t *pte = NULL; - - /* We must align the address, because our caller will run - * set_huge_pte_at() on whatever we return, which writes out - * all of the sub-ptes for the hugepage range. So we have - * to give it the first such sub-pte. - */ - addr &= HPAGE_MASK; pgd = pgd_offset(mm, addr); - pud = pud_alloc(mm, pgd, addr); - if (pud) { - pmd = pmd_alloc(mm, pud, addr); - if (pmd) - pte = pte_alloc_map(mm, NULL, pmd, addr); - } - return pte; + p4d = p4d_offset(pgd, addr); + pud = pud_alloc(mm, p4d, addr); + if (!pud) + return NULL; + if (sz >= PUD_SIZE) + return (pte_t *)pud; + pmd = pmd_alloc(mm, pud, addr); + if (!pmd) + return NULL; + if (sz >= PMD_SIZE) + return (pte_t *)pmd; + return pte_alloc_huge(mm, pmd, addr); } -pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) +pte_t *huge_pte_offset(struct mm_struct *mm, + unsigned long addr, unsigned long sz) { pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; - pte_t *pte = NULL; - - addr &= HPAGE_MASK; pgd = pgd_offset(mm, addr); - if (!pgd_none(*pgd)) { - pud = pud_offset(pgd, addr); - if (!pud_none(*pud)) { - pmd = pmd_offset(pud, addr); - if (!pmd_none(*pmd)) - pte = pte_offset_map(pmd, addr); - } - } - return pte; + if (pgd_none(*pgd)) + return NULL; + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) + return NULL; + pud = pud_offset(p4d, addr); + if (pud_none(*pud)) + return NULL; + if (is_hugetlb_pud(*pud)) + return (pte_t *)pud; + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) + return NULL; + if (is_hugetlb_pmd(*pmd)) + return (pte_t *)pmd; + return pte_offset_huge(pmd, addr); } -int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) +void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t entry) { - return 0; + unsigned int nptes, orig_shift, shift; + unsigned long i, size; + pte_t orig; + + size = huge_tte_to_size(entry); + + shift = PAGE_SHIFT; + if (size >= PUD_SIZE) + shift = PUD_SHIFT; + else if (size >= PMD_SIZE) + shift = PMD_SHIFT; + else + shift = PAGE_SHIFT; + + nptes = size >> shift; + + if (!pte_present(*ptep) && pte_present(entry)) + mm->context.hugetlb_pte_count += nptes; + + addr &= ~(size - 1); + orig = *ptep; + orig_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig); + + for (i = 0; i < nptes; i++) + ptep[i] = __pte(pte_val(entry) + (i << shift)); + + maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift); + /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */ + if (size == HPAGE_SIZE) + maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0, + orig_shift); } void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t entry) + pte_t *ptep, pte_t entry, unsigned long sz) { - int i; - - if (!pte_present(*ptep) && pte_present(entry)) - mm->context.huge_pte_count++; - - addr &= HPAGE_MASK; - for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { - set_pte_at(mm, addr, ptep, entry); - ptep++; - addr += PAGE_SIZE; - pte_val(entry) += PAGE_SIZE; - } + __set_huge_pte_at(mm, addr, ptep, entry); } pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, - pte_t *ptep) + pte_t *ptep, unsigned long sz) { + unsigned int i, nptes, orig_shift, shift; + unsigned long size; pte_t entry; - int i; entry = *ptep; - if (pte_present(entry)) - mm->context.huge_pte_count--; + size = huge_tte_to_size(entry); - addr &= HPAGE_MASK; - - for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { - pte_clear(mm, addr, ptep); - addr += PAGE_SIZE; - ptep++; - } + shift = PAGE_SHIFT; + if (size >= PUD_SIZE) + shift = PUD_SHIFT; + else if (size >= PMD_SIZE) + shift = PMD_SHIFT; + else + shift = PAGE_SHIFT; - return entry; -} + nptes = size >> shift; + orig_shift = pte_none(entry) ? PAGE_SHIFT : huge_tte_to_shift(entry); -struct page *follow_huge_addr(struct mm_struct *mm, - unsigned long address, int write) -{ - return ERR_PTR(-EINVAL); -} + if (pte_present(entry)) + mm->context.hugetlb_pte_count -= nptes; -int pmd_huge(pmd_t pmd) -{ - return 0; -} + addr &= ~(size - 1); + for (i = 0; i < nptes; i++) + ptep[i] = __pte(0UL); -int pud_huge(pud_t pud) -{ - return 0; -} + maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift); + /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */ + if (size == HPAGE_SIZE) + maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0, + orig_shift); -struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, - pmd_t *pmd, int write) -{ - return NULL; + return entry; } diff --git a/arch/sparc/mm/hypersparc.S b/arch/sparc/mm/hypersparc.S index 969f96450f69..6c2521e85a42 100644 --- a/arch/sparc/mm/hypersparc.S +++ b/arch/sparc/mm/hypersparc.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * hypersparc.S: High speed Hypersparc mmu/cache operations. * @@ -9,6 +10,7 @@ #include <asm/asm-offsets.h> #include <asm/asi.h> #include <asm/page.h> +#include <asm/pgtable.h> #include <asm/pgtsrmmu.h> #include <linux/init.h> @@ -292,7 +294,7 @@ hypersparc_flush_tlb_range: cmp %o3, -1 be hypersparc_flush_tlb_range_out #endif - sethi %hi(~((1 << SRMMU_PGDIR_SHIFT) - 1)), %o4 + sethi %hi(~((1 << PGDIR_SHIFT) - 1)), %o4 sta %o3, [%g1] ASI_M_MMUREGS and %o1, %o4, %o1 add %o1, 0x200, %o1 diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c index db6987082805..fdc93dd12c3e 100644 --- a/arch/sparc/mm/init_32.c +++ b/arch/sparc/mm/init_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * linux/arch/sparc/mm/init.c * @@ -21,22 +22,22 @@ #include <linux/initrd.h> #include <linux/init.h> #include <linux/highmem.h> -#include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/pagemap.h> #include <linux/poison.h> #include <linux/gfp.h> #include <asm/sections.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/vaddrs.h> -#include <asm/pgalloc.h> /* bug in asm-generic/tlb.h: check_pgt_cache */ +#include <asm/setup.h> #include <asm/tlb.h> #include <asm/prom.h> #include <asm/leon.h> -unsigned long *sparc_valid_addr_bitmap; -EXPORT_SYMBOL(sparc_valid_addr_bitmap); +#include "mm_32.h" + +static unsigned long *sparc_valid_addr_bitmap; unsigned long phys_base; EXPORT_SYMBOL(phys_base); @@ -52,18 +53,6 @@ extern unsigned int sparc_ramdisk_size; unsigned long highstart_pfn, highend_pfn; -void show_mem(unsigned int filter) -{ - printk("Mem-info:\n"); - show_free_areas(filter); - printk("Free swap: %6ldkB\n", - get_nr_swap_pages() << (PAGE_SHIFT-10)); - printk("%ld pages of RAM\n", totalram_pages); - printk("%ld free pages\n", nr_free_pages()); -} - - -extern unsigned long cmdline_memory_size; unsigned long last_valid_pfn; unsigned long calc_highpages(void) @@ -109,13 +98,46 @@ static unsigned long calc_max_low_pfn(void) return tmp; } +static void __init find_ramdisk(unsigned long end_of_phys_memory) +{ +#ifdef CONFIG_BLK_DEV_INITRD + unsigned long size; + + /* Now have to check initial ramdisk, so that it won't pass + * the end of memory + */ + if (sparc_ramdisk_image) { + if (sparc_ramdisk_image >= (unsigned long)&_end - 2 * PAGE_SIZE) + sparc_ramdisk_image -= KERNBASE; + initrd_start = sparc_ramdisk_image + phys_base; + initrd_end = initrd_start + sparc_ramdisk_size; + if (initrd_end > end_of_phys_memory) { + printk(KERN_CRIT "initrd extends beyond end of memory " + "(0x%016lx > 0x%016lx)\ndisabling initrd\n", + initrd_end, end_of_phys_memory); + initrd_start = 0; + } else { + /* Reserve the initrd image area. */ + size = initrd_end - initrd_start; + memblock_reserve(initrd_start, size); + + initrd_start = (initrd_start - phys_base) + PAGE_OFFSET; + initrd_end = (initrd_end - phys_base) + PAGE_OFFSET; + } + } +#endif +} + unsigned long __init bootmem_init(unsigned long *pages_avail) { - unsigned long bootmap_size, start_pfn; - unsigned long end_of_phys_memory = 0UL; - unsigned long bootmap_pfn, bytes_avail, size; + unsigned long start_pfn, bytes_avail, size; + unsigned long end_of_phys_memory = 0; + unsigned long high_pages = 0; int i; + memblock_set_bottom_up(true); + memblock_allow_resize(); + bytes_avail = 0UL; for (i = 0; sp_banks[i].num_bytes != 0; i++) { end_of_phys_memory = sp_banks[i].base_addr + @@ -132,24 +154,25 @@ unsigned long __init bootmem_init(unsigned long *pages_avail) if (sp_banks[i].num_bytes == 0) { sp_banks[i].base_addr = 0xdeadbeef; } else { + memblock_add(sp_banks[i].base_addr, + sp_banks[i].num_bytes); sp_banks[i+1].num_bytes = 0; sp_banks[i+1].base_addr = 0xdeadbeef; } break; } } + memblock_add(sp_banks[i].base_addr, sp_banks[i].num_bytes); } /* Start with page aligned address of last symbol in kernel - * image. + * image. */ start_pfn = (unsigned long)__pa(PAGE_ALIGN((unsigned long) &_end)); /* Now shift down to get the real physical page frame number. */ start_pfn >>= PAGE_SHIFT; - bootmap_pfn = start_pfn; - max_pfn = end_of_phys_memory >> PAGE_SHIFT; max_low_pfn = max_pfn; @@ -158,85 +181,23 @@ unsigned long __init bootmem_init(unsigned long *pages_avail) if (max_low_pfn > pfn_base + (SRMMU_MAXMEM >> PAGE_SHIFT)) { highstart_pfn = pfn_base + (SRMMU_MAXMEM >> PAGE_SHIFT); max_low_pfn = calc_max_low_pfn(); + high_pages = calc_highpages(); printk(KERN_NOTICE "%ldMB HIGHMEM available.\n", - calc_highpages() >> (20 - PAGE_SHIFT)); + high_pages >> (20 - PAGE_SHIFT)); } -#ifdef CONFIG_BLK_DEV_INITRD - /* Now have to check initial ramdisk, so that bootmap does not overwrite it */ - if (sparc_ramdisk_image) { - if (sparc_ramdisk_image >= (unsigned long)&_end - 2 * PAGE_SIZE) - sparc_ramdisk_image -= KERNBASE; - initrd_start = sparc_ramdisk_image + phys_base; - initrd_end = initrd_start + sparc_ramdisk_size; - if (initrd_end > end_of_phys_memory) { - printk(KERN_CRIT "initrd extends beyond end of memory " - "(0x%016lx > 0x%016lx)\ndisabling initrd\n", - initrd_end, end_of_phys_memory); - initrd_start = 0; - } - if (initrd_start) { - if (initrd_start >= (start_pfn << PAGE_SHIFT) && - initrd_start < (start_pfn << PAGE_SHIFT) + 2 * PAGE_SIZE) - bootmap_pfn = PAGE_ALIGN (initrd_end) >> PAGE_SHIFT; - } - } -#endif - /* Initialize the boot-time allocator. */ - bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap_pfn, pfn_base, - max_low_pfn); - - /* Now register the available physical memory with the - * allocator. - */ - *pages_avail = 0; - for (i = 0; sp_banks[i].num_bytes != 0; i++) { - unsigned long curr_pfn, last_pfn; + find_ramdisk(end_of_phys_memory); - curr_pfn = sp_banks[i].base_addr >> PAGE_SHIFT; - if (curr_pfn >= max_low_pfn) - break; - - last_pfn = (sp_banks[i].base_addr + sp_banks[i].num_bytes) >> PAGE_SHIFT; - if (last_pfn > max_low_pfn) - last_pfn = max_low_pfn; - - /* - * .. finally, did all the rounding and playing - * around just make the area go away? - */ - if (last_pfn <= curr_pfn) - continue; - - size = (last_pfn - curr_pfn) << PAGE_SHIFT; - *pages_avail += last_pfn - curr_pfn; - - free_bootmem(sp_banks[i].base_addr, size); - } - -#ifdef CONFIG_BLK_DEV_INITRD - if (initrd_start) { - /* Reserve the initrd image area. */ - size = initrd_end - initrd_start; - reserve_bootmem(initrd_start, size, BOOTMEM_DEFAULT); - *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT; - - initrd_start = (initrd_start - phys_base) + PAGE_OFFSET; - initrd_end = (initrd_end - phys_base) + PAGE_OFFSET; - } -#endif /* Reserve the kernel text/data/bss. */ size = (start_pfn << PAGE_SHIFT) - phys_base; - reserve_bootmem(phys_base, size, BOOTMEM_DEFAULT); - *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT; + memblock_reserve(phys_base, size); + memblock_add(phys_base, size); - /* Reserve the bootmem map. We do not account for it - * in pages_avail because we will release that memory - * in free_all_bootmem. - */ - size = bootmap_size; - reserve_bootmem((bootmap_pfn << PAGE_SHIFT), size, BOOTMEM_DEFAULT); - *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT; + size = memblock_phys_mem_size() - memblock_reserved_size(); + *pages_avail = (size >> PAGE_SHIFT) - high_pages; + + /* Only allow low memory to be allocated via memblock allocation */ + memblock_set_current_limit(max_low_pfn << PAGE_SHIFT); return max_pfn; } @@ -246,9 +207,6 @@ unsigned long __init bootmem_init(unsigned long *pages_avail) * init routine based upon the Sun model type on the Sparc. * */ -extern void srmmu_paging_init(void); -extern void device_scan(void); - void __init paging_init(void) { srmmu_paging_init(); @@ -274,19 +232,7 @@ static void __init taint_real_pages(void) } } -static void map_high_region(unsigned long start_pfn, unsigned long end_pfn) -{ - unsigned long tmp; - -#ifdef CONFIG_DEBUG_HIGHMEM - printk("mapping high region %08lx - %08lx\n", start_pfn, end_pfn); -#endif - - for (tmp = start_pfn; tmp < end_pfn; tmp++) - free_highmem_page(pfn_to_page(tmp)); -} - -void __init mem_init(void) +void __init arch_mm_preinit(void) { int i; @@ -302,12 +248,12 @@ void __init mem_init(void) /* Saves us work later. */ - memset((void *)&empty_zero_page, 0, PAGE_SIZE); + memset((void *)empty_zero_page, 0, PAGE_SIZE); i = last_valid_pfn >> ((20 - PAGE_SHIFT) + 5); i += 1; sparc_valid_addr_bitmap = (unsigned long *) - __alloc_bootmem(i << 2, SMP_CACHE_BYTES, 0UL); + memblock_alloc(i << 2, SMP_CACHE_BYTES); if (sparc_valid_addr_bitmap == NULL) { prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n"); @@ -316,45 +262,42 @@ void __init mem_init(void) memset(sparc_valid_addr_bitmap, 0, i << 2); taint_real_pages(); - - max_mapnr = last_valid_pfn - pfn_base; - high_memory = __va(max_low_pfn << PAGE_SHIFT); - free_all_bootmem(); - - for (i = 0; sp_banks[i].num_bytes != 0; i++) { - unsigned long start_pfn = sp_banks[i].base_addr >> PAGE_SHIFT; - unsigned long end_pfn = (sp_banks[i].base_addr + sp_banks[i].num_bytes) >> PAGE_SHIFT; - - if (end_pfn <= highstart_pfn) - continue; - - if (start_pfn < highstart_pfn) - start_pfn = highstart_pfn; - - map_high_region(start_pfn, end_pfn); - } - - mem_init_print_info(NULL); } -void free_initmem (void) +void sparc_flush_page_to_ram(struct page *page) { - free_initmem_default(POISON_FREE_INITMEM); -} + unsigned long vaddr = (unsigned long)page_address(page); -#ifdef CONFIG_BLK_DEV_INITRD -void free_initrd_mem(unsigned long start, unsigned long end) -{ - free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM, - "initrd"); + __flush_page_to_ram(vaddr); } -#endif +EXPORT_SYMBOL(sparc_flush_page_to_ram); -void sparc_flush_page_to_ram(struct page *page) +void sparc_flush_folio_to_ram(struct folio *folio) { - unsigned long vaddr = (unsigned long)page_address(page); + unsigned long vaddr = (unsigned long)folio_address(folio); + unsigned int i, nr = folio_nr_pages(folio); - if (vaddr) - __flush_page_to_ram(vaddr); + for (i = 0; i < nr; i++) + __flush_page_to_ram(vaddr + i * PAGE_SIZE); } -EXPORT_SYMBOL(sparc_flush_page_to_ram); +EXPORT_SYMBOL(sparc_flush_folio_to_ram); + +static const pgprot_t protection_map[16] = { + [VM_NONE] = PAGE_NONE, + [VM_READ] = PAGE_READONLY, + [VM_WRITE] = PAGE_COPY, + [VM_WRITE | VM_READ] = PAGE_COPY, + [VM_EXEC] = PAGE_READONLY, + [VM_EXEC | VM_READ] = PAGE_READONLY, + [VM_EXEC | VM_WRITE] = PAGE_COPY, + [VM_EXEC | VM_WRITE | VM_READ] = PAGE_COPY, + [VM_SHARED] = PAGE_NONE, + [VM_SHARED | VM_READ] = PAGE_READONLY, + [VM_SHARED | VM_WRITE] = PAGE_SHARED, + [VM_SHARED | VM_WRITE | VM_READ] = PAGE_SHARED, + [VM_SHARED | VM_EXEC] = PAGE_READONLY, + [VM_SHARED | VM_EXEC | VM_READ] = PAGE_READONLY, + [VM_SHARED | VM_EXEC | VM_WRITE] = PAGE_SHARED, + [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_SHARED +}; +DECLARE_VM_GET_PAGE_PROT diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index a9c42a7ffb6a..df9f7c444c39 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * arch/sparc64/mm/init.c * @@ -5,12 +6,12 @@ * Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ -#include <linux/module.h> +#include <linux/extable.h> #include <linux/kernel.h> #include <linux/sched.h> #include <linux/string.h> #include <linux/init.h> -#include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/mm.h> #include <linux/hugetlb.h> #include <linux/initrd.h> @@ -22,19 +23,19 @@ #include <linux/kprobes.h> #include <linux/cache.h> #include <linux/sort.h> +#include <linux/ioport.h> #include <linux/percpu.h> -#include <linux/memblock.h> #include <linux/mmzone.h> #include <linux/gfp.h> +#include <linux/bootmem_info.h> #include <asm/head.h> #include <asm/page.h> #include <asm/pgalloc.h> -#include <asm/pgtable.h> #include <asm/oplib.h> #include <asm/iommu.h> #include <asm/io.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/mmu_context.h> #include <asm/tlbflush.h> #include <asm/dma.h> @@ -47,11 +48,13 @@ #include <asm/prom.h> #include <asm/mdesc.h> #include <asm/cpudata.h> +#include <asm/setup.h> #include <asm/irq.h> #include "init_64.h" unsigned long kern_linear_pte_xor[4] __read_mostly; +static unsigned long page_cache4v_flag; /* A bitmap, two bits for every 256MB of physical memory. These two * bits determine what page size we use for kernel linear @@ -73,7 +76,6 @@ unsigned long kern_linear_pte_xor[4] __read_mostly; * 'cpu' properties, but we need to have this table setup before the * MDESC is initialized. */ -unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)]; #ifndef CONFIG_DEBUG_PAGEALLOC /* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings. @@ -82,14 +84,17 @@ unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)]; */ extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES]; #endif +extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; static unsigned long cpu_pgsz_mask; -#define MAX_BANKS 32 +#define MAX_BANKS 1024 static struct linux_prom64_registers pavail[MAX_BANKS]; static int pavail_ents; +u64 numa_latency[MAX_NUMNODES][MAX_NUMNODES]; + static int cmp_p64(const void *a, const void *b) { const struct linux_prom64_registers *x = a, *y = b; @@ -163,10 +168,6 @@ static void __init read_obp_memory(const char *property, cmp_p64, NULL); } -unsigned long sparc64_valid_addr_bitmap[VALID_ADDR_BITMAP_BYTES / - sizeof(unsigned long)]; -EXPORT_SYMBOL(sparc64_valid_addr_bitmap); - /* Kernel physical address base and size in bytes. */ unsigned long kern_base __read_mostly; unsigned long kern_size __read_mostly; @@ -194,21 +195,26 @@ atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0); #endif #endif -inline void flush_dcache_page_impl(struct page *page) +inline void flush_dcache_folio_impl(struct folio *folio) { + unsigned int i, nr = folio_nr_pages(folio); + BUG_ON(tlb_type == hypervisor); #ifdef CONFIG_DEBUG_DCFLUSH atomic_inc(&dcpage_flushes); #endif #ifdef DCACHE_ALIASING_POSSIBLE - __flush_dcache_page(page_address(page), - ((tlb_type == spitfire) && - page_mapping(page) != NULL)); + for (i = 0; i < nr; i++) + __flush_dcache_page(folio_address(folio) + i * PAGE_SIZE, + ((tlb_type == spitfire) && + folio_flush_mapping(folio) != NULL)); #else - if (page_mapping(page) != NULL && - tlb_type == spitfire) - __flush_icache_page(__pa(page_address(page))); + if (folio_flush_mapping(folio) != NULL && + tlb_type == spitfire) { + for (i = 0; i < nr; i++) + __flush_icache_page((pfn + i) * PAGE_SIZE); + } #endif } @@ -217,10 +223,10 @@ inline void flush_dcache_page_impl(struct page *page) #define PG_dcache_cpu_mask \ ((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL) -#define dcache_dirty_cpu(page) \ - (((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask) +#define dcache_dirty_cpu(folio) \ + (((folio)->flags.f >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask) -static inline void set_dcache_dirty(struct page *page, int this_cpu) +static inline void set_dcache_dirty(struct folio *folio, int this_cpu) { unsigned long mask = this_cpu; unsigned long non_cpu_bits; @@ -237,11 +243,11 @@ static inline void set_dcache_dirty(struct page *page, int this_cpu) "bne,pn %%xcc, 1b\n\t" " nop" : /* no outputs */ - : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags) + : "r" (mask), "r" (non_cpu_bits), "r" (&folio->flags.f) : "g1", "g7"); } -static inline void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu) +static inline void clear_dcache_dirty_cpu(struct folio *folio, unsigned long cpu) { unsigned long mask = (1UL << PG_dcache_dirty); @@ -259,7 +265,7 @@ static inline void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu) " nop\n" "2:" : /* no outputs */ - : "r" (cpu), "r" (mask), "r" (&page->flags), + : "r" (cpu), "r" (mask), "r" (&folio->flags.f), "i" (PG_dcache_cpu_mask), "i" (PG_dcache_cpu_shift) : "g1", "g7"); @@ -283,9 +289,10 @@ static void flush_dcache(unsigned long pfn) page = pfn_to_page(pfn); if (page) { + struct folio *folio = page_folio(page); unsigned long pg_flags; - pg_flags = page->flags; + pg_flags = folio->flags.f; if (pg_flags & (1UL << PG_dcache_dirty)) { int cpu = ((pg_flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask); @@ -295,11 +302,11 @@ static void flush_dcache(unsigned long pfn) * in the SMP case. */ if (cpu == this_cpu) - flush_dcache_page_impl(page); + flush_dcache_folio_impl(folio); else - smp_flush_dcache_page_impl(page, cpu); + smp_flush_dcache_folio_impl(folio, cpu); - clear_dcache_dirty_cpu(page, cpu); + clear_dcache_dirty_cpu(folio, cpu); put_cpu(); } @@ -323,23 +330,78 @@ static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_inde tsb_insert(tsb, tag, tte); } -#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) -static inline bool is_hugetlb_pte(pte_t pte) +#ifdef CONFIG_HUGETLB_PAGE +static int __init hugetlbpage_init(void) { - if ((tlb_type == hypervisor && - (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) || - (tlb_type != hypervisor && - (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U)) - return true; - return false; + hugetlb_add_hstate(HPAGE_64K_SHIFT - PAGE_SHIFT); + hugetlb_add_hstate(HPAGE_SHIFT - PAGE_SHIFT); + hugetlb_add_hstate(HPAGE_256MB_SHIFT - PAGE_SHIFT); + hugetlb_add_hstate(HPAGE_2GB_SHIFT - PAGE_SHIFT); + + return 0; } -#endif -void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) +arch_initcall(hugetlbpage_init); + +static void __init pud_huge_patch(void) +{ + struct pud_huge_patch_entry *p; + unsigned long addr; + + p = &__pud_huge_patch; + addr = p->addr; + *(unsigned int *)addr = p->insn; + + __asm__ __volatile__("flush %0" : : "r" (addr)); +} + +bool __init arch_hugetlb_valid_size(unsigned long size) +{ + unsigned int hugepage_shift = ilog2(size); + unsigned short hv_pgsz_idx; + unsigned int hv_pgsz_mask; + + switch (hugepage_shift) { + case HPAGE_16GB_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_16GB; + hv_pgsz_idx = HV_PGSZ_IDX_16GB; + pud_huge_patch(); + break; + case HPAGE_2GB_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_2GB; + hv_pgsz_idx = HV_PGSZ_IDX_2GB; + break; + case HPAGE_256MB_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_256MB; + hv_pgsz_idx = HV_PGSZ_IDX_256MB; + break; + case HPAGE_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_4MB; + hv_pgsz_idx = HV_PGSZ_IDX_4MB; + break; + case HPAGE_64K_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_64K; + hv_pgsz_idx = HV_PGSZ_IDX_64K; + break; + default: + hv_pgsz_mask = 0; + } + + if ((hv_pgsz_mask & cpu_pgsz_mask) == 0U) + return false; + + return true; +} +#endif /* CONFIG_HUGETLB_PAGE */ + +void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, + unsigned long address, pte_t *ptep, unsigned int nr) { struct mm_struct *mm; unsigned long flags; + bool is_huge_tsb; pte_t pte = *ptep; + unsigned int i; if (tlb_type != hypervisor) { unsigned long pfn = pte_pfn(pte); @@ -350,22 +412,57 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t * mm = vma->vm_mm; + /* Don't insert a non-valid PTE into the TSB, we'll deadlock. */ + if (!pte_accessible(mm, pte)) + return; + spin_lock_irqsave(&mm->context.lock, flags); + is_huge_tsb = false; #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (mm->context.huge_pte_count && is_hugetlb_pte(pte)) - __update_mmu_tsb_insert(mm, MM_TSB_HUGE, HPAGE_SHIFT, - address, pte_val(pte)); - else + if (mm->context.hugetlb_pte_count || mm->context.thp_pte_count) { + unsigned long hugepage_size = PAGE_SIZE; + + if (is_vm_hugetlb_page(vma)) + hugepage_size = huge_page_size(hstate_vma(vma)); + + if (hugepage_size >= PUD_SIZE) { + unsigned long mask = 0x1ffc00000UL; + + /* Transfer bits [32:22] from address to resolve + * at 4M granularity. + */ + pte_val(pte) &= ~mask; + pte_val(pte) |= (address & mask); + } else if (hugepage_size >= PMD_SIZE) { + /* We are fabricating 8MB pages using 4MB + * real hw pages. + */ + pte_val(pte) |= (address & (1UL << REAL_HPAGE_SHIFT)); + } + + if (hugepage_size >= PMD_SIZE) { + __update_mmu_tsb_insert(mm, MM_TSB_HUGE, + REAL_HPAGE_SHIFT, address, pte_val(pte)); + is_huge_tsb = true; + } + } #endif - __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT, - address, pte_val(pte)); + if (!is_huge_tsb) { + for (i = 0; i < nr; i++) { + __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT, + address, pte_val(pte)); + address += PAGE_SIZE; + pte_val(pte) += PAGE_SIZE; + } + } spin_unlock_irqrestore(&mm->context.lock, flags); } -void flush_dcache_page(struct page *page) +void flush_dcache_folio(struct folio *folio) { + unsigned long pfn = folio_pfn(folio); struct address_space *mapping; int this_cpu; @@ -376,35 +473,35 @@ void flush_dcache_page(struct page *page) * is merely the zero page. The 'bigcore' testcase in GDB * causes this case to run millions of times. */ - if (page == ZERO_PAGE(0)) + if (is_zero_pfn(pfn)) return; this_cpu = get_cpu(); - mapping = page_mapping(page); + mapping = folio_flush_mapping(folio); if (mapping && !mapping_mapped(mapping)) { - int dirty = test_bit(PG_dcache_dirty, &page->flags); + bool dirty = test_bit(PG_dcache_dirty, &folio->flags.f); if (dirty) { - int dirty_cpu = dcache_dirty_cpu(page); + int dirty_cpu = dcache_dirty_cpu(folio); if (dirty_cpu == this_cpu) goto out; - smp_flush_dcache_page_impl(page, dirty_cpu); + smp_flush_dcache_folio_impl(folio, dirty_cpu); } - set_dcache_dirty(page, this_cpu); + set_dcache_dirty(folio, this_cpu); } else { - /* We could delay the flush for the !page_mapping + /* We could delay the flush for the !folio_mapping * case too. But that case is for exec env/arg * pages and those are %99 certainly going to get * faulted into the tlb (and thus flushed) anyways. */ - flush_dcache_page_impl(page); + flush_dcache_folio_impl(folio); } out: put_cpu(); } -EXPORT_SYMBOL(flush_dcache_page); +EXPORT_SYMBOL(flush_dcache_folio); void __kprobes flush_icache_range(unsigned long start, unsigned long end) { @@ -421,10 +518,7 @@ void __kprobes flush_icache_range(unsigned long start, unsigned long end) if (kaddr >= PAGE_OFFSET) paddr = kaddr & mask; else { - pgd_t *pgdp = pgd_offset_k(kaddr); - pud_t *pudp = pud_offset(pgdp, kaddr); - pmd_t *pmdp = pmd_offset(pudp, kaddr); - pte_t *ptep = pte_offset_kernel(pmdp, kaddr); + pte_t *ptep = virt_to_kpte(kaddr); paddr = pte_val(*ptep) & mask; } @@ -588,7 +682,7 @@ static void __init remap_kernel(void) int i, tlb_ent = sparc64_highest_locked_tlbent(); tte_vaddr = (unsigned long) KERNBASE; - phys_page = (prom_boot_mapping_phys_low >> 22UL) << 22UL; + phys_page = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB; tte_data = kern_large_tte(phys_page); kern_locked_tte_data = tte_data; @@ -629,9 +723,10 @@ static void __init inherit_prom_mappings(void) void prom_world(int enter) { - if (!enter) - set_fs(get_fs()); - + /* + * No need to change the address space any more, just flush + * the register windows + */ __asm__ __volatile__("flushw"); } @@ -662,10 +757,58 @@ EXPORT_SYMBOL(__flush_dcache_range); /* get_new_mmu_context() uses "cache + 1". */ DEFINE_SPINLOCK(ctx_alloc_lock); -unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1; +unsigned long tlb_context_cache = CTX_FIRST_VERSION; #define MAX_CTX_NR (1UL << CTX_NR_BITS) #define CTX_BMAP_SLOTS BITS_TO_LONGS(MAX_CTX_NR) DECLARE_BITMAP(mmu_context_bmap, MAX_CTX_NR); +DEFINE_PER_CPU(struct mm_struct *, per_cpu_secondary_mm) = {0}; + +static void mmu_context_wrap(void) +{ + unsigned long old_ver = tlb_context_cache & CTX_VERSION_MASK; + unsigned long new_ver, new_ctx, old_ctx; + struct mm_struct *mm; + int cpu; + + bitmap_zero(mmu_context_bmap, 1 << CTX_NR_BITS); + + /* Reserve kernel context */ + set_bit(0, mmu_context_bmap); + + new_ver = (tlb_context_cache & CTX_VERSION_MASK) + CTX_FIRST_VERSION; + if (unlikely(new_ver == 0)) + new_ver = CTX_FIRST_VERSION; + tlb_context_cache = new_ver; + + /* + * Make sure that any new mm that are added into per_cpu_secondary_mm, + * are going to go through get_new_mmu_context() path. + */ + mb(); + + /* + * Updated versions to current on those CPUs that had valid secondary + * contexts + */ + for_each_online_cpu(cpu) { + /* + * If a new mm is stored after we took this mm from the array, + * it will go into get_new_mmu_context() path, because we + * already bumped the version in tlb_context_cache. + */ + mm = per_cpu(per_cpu_secondary_mm, cpu); + + if (unlikely(!mm || mm == &init_mm)) + continue; + + old_ctx = mm->context.sparc64_ctx_val; + if (likely((old_ctx & CTX_VERSION_MASK) == old_ver)) { + new_ctx = (old_ctx & ~CTX_VERSION_MASK) | new_ver; + set_bit(new_ctx & CTX_NR_MASK, mmu_context_bmap); + mm->context.sparc64_ctx_val = new_ctx; + } + } +} /* Caller does TLB context flushing on local CPU if necessary. * The caller also ensures that CTX_VALID(mm->context) is false. @@ -681,48 +824,30 @@ void get_new_mmu_context(struct mm_struct *mm) { unsigned long ctx, new_ctx; unsigned long orig_pgsz_bits; - int new_version; spin_lock(&ctx_alloc_lock); +retry: + /* wrap might have happened, test again if our context became valid */ + if (unlikely(CTX_VALID(mm->context))) + goto out; orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK); ctx = (tlb_context_cache + 1) & CTX_NR_MASK; new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx); - new_version = 0; if (new_ctx >= (1 << CTX_NR_BITS)) { new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1); if (new_ctx >= ctx) { - int i; - new_ctx = (tlb_context_cache & CTX_VERSION_MASK) + - CTX_FIRST_VERSION; - if (new_ctx == 1) - new_ctx = CTX_FIRST_VERSION; - - /* Don't call memset, for 16 entries that's just - * plain silly... - */ - mmu_context_bmap[0] = 3; - mmu_context_bmap[1] = 0; - mmu_context_bmap[2] = 0; - mmu_context_bmap[3] = 0; - for (i = 4; i < CTX_BMAP_SLOTS; i += 4) { - mmu_context_bmap[i + 0] = 0; - mmu_context_bmap[i + 1] = 0; - mmu_context_bmap[i + 2] = 0; - mmu_context_bmap[i + 3] = 0; - } - new_version = 1; - goto out; + mmu_context_wrap(); + goto retry; } } + if (mm->context.sparc64_ctx_val) + cpumask_clear(mm_cpumask(mm)); mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63)); new_ctx |= (tlb_context_cache & CTX_VERSION_MASK); -out: tlb_context_cache = new_ctx; mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits; +out: spin_unlock(&ctx_alloc_lock); - - if (unlikely(new_version)) - smp_new_mmu_context_version(); } static int numa_enabled = 1; @@ -789,16 +914,26 @@ static void __init find_ramdisk(unsigned long phys_base) struct node_mem_mask { unsigned long mask; - unsigned long val; + unsigned long match; }; static struct node_mem_mask node_masks[MAX_NUMNODES]; static int num_node_masks; +#ifdef CONFIG_NUMA + +struct mdesc_mlgroup { + u64 node; + u64 latency; + u64 match; + u64 mask; +}; + +static struct mdesc_mlgroup *mlgroups; +static int num_mlgroups; + int numa_cpu_lookup_table[NR_CPUS]; cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES]; -#ifdef CONFIG_NEED_MULTIPLE_NODES - struct mdesc_mblock { u64 base; u64 size; @@ -807,52 +942,128 @@ struct mdesc_mblock { static struct mdesc_mblock *mblocks; static int num_mblocks; -static unsigned long ra_to_pa(unsigned long addr) +static struct mdesc_mblock * __init addr_to_mblock(unsigned long addr) { + struct mdesc_mblock *m = NULL; int i; for (i = 0; i < num_mblocks; i++) { - struct mdesc_mblock *m = &mblocks[i]; + m = &mblocks[i]; if (addr >= m->base && addr < (m->base + m->size)) { - addr += m->offset; break; } } - return addr; + + return m; } -static int find_node(unsigned long addr) +static u64 __init memblock_nid_range_sun4u(u64 start, u64 end, int *nid) { - int i; + int prev_nid, new_nid; - addr = ra_to_pa(addr); - for (i = 0; i < num_node_masks; i++) { - struct node_mem_mask *p = &node_masks[i]; + prev_nid = NUMA_NO_NODE; + for ( ; start < end; start += PAGE_SIZE) { + for (new_nid = 0; new_nid < num_node_masks; new_nid++) { + struct node_mem_mask *p = &node_masks[new_nid]; + + if ((start & p->mask) == p->match) { + if (prev_nid == NUMA_NO_NODE) + prev_nid = new_nid; + break; + } + } + + if (new_nid == num_node_masks) { + prev_nid = 0; + WARN_ONCE(1, "addr[%Lx] doesn't match a NUMA node rule. Some memory will be owned by node 0.", + start); + break; + } - if ((addr & p->mask) == p->val) - return i; + if (prev_nid != new_nid) + break; } - return -1; + *nid = prev_nid; + + return start > end ? end : start; } -static u64 memblock_nid_range(u64 start, u64 end, int *nid) +static u64 __init memblock_nid_range(u64 start, u64 end, int *nid) { - *nid = find_node(start); - start += PAGE_SIZE; - while (start < end) { - int n = find_node(start); + u64 ret_end, pa_start, m_mask, m_match, m_end; + struct mdesc_mblock *mblock; + int _nid, i; + + if (tlb_type != hypervisor) + return memblock_nid_range_sun4u(start, end, nid); + + mblock = addr_to_mblock(start); + if (!mblock) { + WARN_ONCE(1, "memblock_nid_range: Can't find mblock addr[%Lx]", + start); + + _nid = 0; + ret_end = end; + goto done; + } - if (n != *nid) + pa_start = start + mblock->offset; + m_match = 0; + m_mask = 0; + + for (_nid = 0; _nid < num_node_masks; _nid++) { + struct node_mem_mask *const m = &node_masks[_nid]; + + if ((pa_start & m->mask) == m->match) { + m_match = m->match; + m_mask = m->mask; break; - start += PAGE_SIZE; + } } - if (start > end) - start = end; + if (num_node_masks == _nid) { + /* We could not find NUMA group, so default to 0, but lets + * search for latency group, so we could calculate the correct + * end address that we return + */ + _nid = 0; + + for (i = 0; i < num_mlgroups; i++) { + struct mdesc_mlgroup *const m = &mlgroups[i]; - return start; + if ((pa_start & m->mask) == m->match) { + m_match = m->match; + m_mask = m->mask; + break; + } + } + + if (i == num_mlgroups) { + WARN_ONCE(1, "memblock_nid_range: Can't find latency group addr[%Lx]", + start); + + ret_end = end; + goto done; + } + } + + /* + * Each latency group has match and mask, and each memory block has an + * offset. An address belongs to a latency group if its address matches + * the following formula: ((addr + offset) & mask) == match + * It is, however, slow to check every single page if it matches a + * particular latency group. As optimization we calculate end value by + * using bit arithmetics. + */ + m_end = m_match + (1ul << __ffs(m_mask)) - mblock->offset; + m_end += pa_start & ~((1ul << fls64(m_mask)) - 1); + ret_end = m_end > end ? end : m_end; + +done: + *nid = _nid; + return ret_end; } #endif @@ -864,16 +1075,9 @@ static void __init allocate_node_data(int nid) { struct pglist_data *p; unsigned long start_pfn, end_pfn; -#ifdef CONFIG_NEED_MULTIPLE_NODES - unsigned long paddr; - paddr = memblock_alloc_try_nid(sizeof(struct pglist_data), SMP_CACHE_BYTES, nid); - if (!paddr) { - prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid); - prom_halt(); - } - NODE_DATA(nid) = __va(paddr); - memset(NODE_DATA(nid), 0, sizeof(struct pglist_data)); +#ifdef CONFIG_NUMA + alloc_node_data(nid); NODE_DATA(nid)->node_id = nid; #endif @@ -887,34 +1091,28 @@ static void __init allocate_node_data(int nid) static void init_node_masks_nonnuma(void) { +#ifdef CONFIG_NUMA int i; +#endif numadbg("Initializing tables for non-numa.\n"); - node_masks[0].mask = node_masks[0].val = 0; + node_masks[0].mask = 0; + node_masks[0].match = 0; num_node_masks = 1; +#ifdef CONFIG_NUMA for (i = 0; i < NR_CPUS; i++) numa_cpu_lookup_table[i] = 0; cpumask_setall(&numa_cpumask_lookup_table[0]); +#endif } -#ifdef CONFIG_NEED_MULTIPLE_NODES -struct pglist_data *node_data[MAX_NUMNODES]; +#ifdef CONFIG_NUMA EXPORT_SYMBOL(numa_cpu_lookup_table); EXPORT_SYMBOL(numa_cpumask_lookup_table); -EXPORT_SYMBOL(node_data); - -struct mdesc_mlgroup { - u64 node; - u64 latency; - u64 match; - u64 mask; -}; -static struct mdesc_mlgroup *mlgroups; -static int num_mlgroups; static int scan_pio_for_cfg_handle(struct mdesc_handle *md, u64 pio, u32 cfg_handle) @@ -987,7 +1185,7 @@ int of_node_to_nid(struct device_node *dp) md = mdesc_grab(); count = 0; - nid = -1; + nid = NUMA_NO_NODE; mdesc_for_each_node_by_name(md, grp, "group") { if (!scan_arcs_for_cfg_handle(md, grp, cfg_handle)) { nid = count; @@ -1003,14 +1201,14 @@ int of_node_to_nid(struct device_node *dp) static void __init add_node_ranges(void) { - struct memblock_region *reg; + phys_addr_t start, end; + unsigned long prev_max; + u64 i; - for_each_memblock(memory, reg) { - unsigned long size = reg->size; - unsigned long start, end; +memblock_resized: + prev_max = memblock.memory.max; - start = reg->base; - end = start + size; + for_each_mem_range(i, &start, &end) { while (start < end) { unsigned long this_end; int nid; @@ -1018,10 +1216,13 @@ static void __init add_node_ranges(void) this_end = memblock_nid_range(start, end, &nid); numadbg("Setting memblock NUMA node nid[%d] " - "start[%lx] end[%lx]\n", + "start[%llx] end[%lx]\n", nid, start, this_end); - memblock_set_node(start, this_end - start, nid); + memblock_set_node(start, this_end - start, + &memblock.memory, nid); + if (memblock.memory.max != prev_max) + goto memblock_resized; start = this_end; } } @@ -1038,8 +1239,8 @@ static int __init grab_mlgroups(struct mdesc_handle *md) if (!count) return -ENOENT; - paddr = memblock_alloc(count * sizeof(struct mdesc_mlgroup), - SMP_CACHE_BYTES); + paddr = memblock_phys_alloc(count * sizeof(struct mdesc_mlgroup), + SMP_CACHE_BYTES); if (!paddr) return -ENOMEM; @@ -1079,8 +1280,8 @@ static int __init grab_mblocks(struct mdesc_handle *md) if (!count) return -ENOENT; - paddr = memblock_alloc(count * sizeof(struct mdesc_mblock), - SMP_CACHE_BYTES); + paddr = memblock_phys_alloc(count * sizeof(struct mdesc_mblock), + SMP_CACHE_BYTES); if (!paddr) return -ENOMEM; @@ -1146,6 +1347,49 @@ static struct mdesc_mlgroup * __init find_mlgroup(u64 node) return NULL; } +int __node_distance(int from, int to) +{ + if ((from >= MAX_NUMNODES) || (to >= MAX_NUMNODES)) { + pr_warn("Returning default NUMA distance value for %d->%d\n", + from, to); + return (from == to) ? LOCAL_DISTANCE : REMOTE_DISTANCE; + } + return numa_latency[from][to]; +} +EXPORT_SYMBOL(__node_distance); + +static int __init find_best_numa_node_for_mlgroup(struct mdesc_mlgroup *grp) +{ + int i; + + for (i = 0; i < MAX_NUMNODES; i++) { + struct node_mem_mask *n = &node_masks[i]; + + if ((grp->mask == n->mask) && (grp->match == n->match)) + break; + } + return i; +} + +static void __init find_numa_latencies_for_group(struct mdesc_handle *md, + u64 grp, int index) +{ + u64 arc; + + mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) { + int tnode; + u64 target = mdesc_arc_target(md, arc); + struct mdesc_mlgroup *m = find_mlgroup(target); + + if (!m) + continue; + tnode = find_best_numa_node_for_mlgroup(m); + if (tnode == MAX_NUMNODES) + continue; + numa_latency[index][tnode] = m->latency; + } +} + static int __init numa_attach_mlgroup(struct mdesc_handle *md, u64 grp, int index) { @@ -1176,10 +1420,10 @@ static int __init numa_attach_mlgroup(struct mdesc_handle *md, u64 grp, n = &node_masks[num_node_masks++]; n->mask = candidate->mask; - n->val = candidate->match; + n->match = candidate->match; - numadbg("NUMA NODE[%d]: mask[%lx] val[%lx] (latency[%llx])\n", - index, n->mask, n->val, candidate->latency); + numadbg("NUMA NODE[%d]: mask[%lx] match[%lx] (latency[%llx])\n", + index, n->mask, n->match, candidate->latency); return 0; } @@ -1209,7 +1453,7 @@ static int __init numa_parse_mdesc_group(struct mdesc_handle *md, u64 grp, static int __init numa_parse_mdesc(void) { struct mdesc_handle *md = mdesc_grab(); - int i, err, count; + int i, j, err, count; u64 node; node = mdesc_node_by_name(md, MDESC_NODE_NULL, "latency-groups"); @@ -1234,6 +1478,23 @@ static int __init numa_parse_mdesc(void) count++; } + count = 0; + mdesc_for_each_node_by_name(md, node, "group") { + find_numa_latencies_for_group(md, node, count); + count++; + } + + /* Normalize numa latency matrix according to ACPI SLIT spec. */ + for (i = 0; i < MAX_NUMNODES; i++) { + u64 self_latency = numa_latency[i][i]; + + for (j = 0; j < MAX_NUMNODES; j++) { + numa_latency[i][j] = + (numa_latency[i][j] * LOCAL_DISTANCE) / + self_latency; + } + } + add_node_ranges(); for (i = 0; i < num_node_masks; i++) { @@ -1259,7 +1520,7 @@ static int __init numa_parse_jbus(void) numa_cpu_lookup_table[cpu] = index; cpumask_copy(&numa_cpumask_lookup_table[index], cpumask_of(cpu)); node_masks[index].mask = ~((1UL << 36UL) - 1UL); - node_masks[index].val = cpu << 36UL; + node_masks[index].match = cpu << 36UL; index++; } @@ -1290,10 +1551,18 @@ static int __init numa_parse_sun4u(void) static int __init bootmem_init_numa(void) { + int i, j; int err = -1; numadbg("bootmem_init_numa()\n"); + /* Some sane defaults for numa latency values */ + for (i = 0; i < MAX_NUMNODES; i++) { + for (j = 0; j < MAX_NUMNODES; j++) + numa_latency[i][j] = (i == j) ? + LOCAL_DISTANCE : REMOTE_DISTANCE; + } + if (numa_enabled) { if (tlb_type == hypervisor) err = numa_parse_mdesc(); @@ -1325,7 +1594,7 @@ static void __init bootmem_init_nonnuma(void) (top_of_ram - total_ram) >> 20); init_node_masks_nonnuma(); - memblock_set_node(0, (phys_addr_t)ULLONG_MAX, 0); + memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0); allocate_node_data(0); node_set_online(0); } @@ -1346,7 +1615,6 @@ static unsigned long __init bootmem_init(unsigned long phys_base) /* XXX cpu notifier XXX */ - sparse_memory_present_with_active_regions(MAX_NUMNODES); sparse_init(); return end_pfn; @@ -1355,9 +1623,148 @@ static unsigned long __init bootmem_init(unsigned long phys_base) static struct linux_prom64_registers pall[MAX_BANKS] __initdata; static int pall_ents __initdata; -#ifdef CONFIG_DEBUG_PAGEALLOC +static unsigned long max_phys_bits = 40; + +bool kern_addr_valid(unsigned long addr) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + if ((long)addr < 0L) { + unsigned long pa = __pa(addr); + + if ((pa >> max_phys_bits) != 0UL) + return false; + + return pfn_valid(pa >> PAGE_SHIFT); + } + + if (addr >= (unsigned long) KERNBASE && + addr < (unsigned long)&_end) + return true; + + pgd = pgd_offset_k(addr); + if (pgd_none(*pgd)) + return false; + + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) + return false; + + pud = pud_offset(p4d, addr); + if (pud_none(*pud)) + return false; + + if (pud_leaf(*pud)) + return pfn_valid(pud_pfn(*pud)); + + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) + return false; + + if (pmd_leaf(*pmd)) + return pfn_valid(pmd_pfn(*pmd)); + + pte = pte_offset_kernel(pmd, addr); + if (pte_none(*pte)) + return false; + + return pfn_valid(pte_pfn(*pte)); +} + +static unsigned long __ref kernel_map_hugepud(unsigned long vstart, + unsigned long vend, + pud_t *pud) +{ + const unsigned long mask16gb = (1UL << 34) - 1UL; + u64 pte_val = vstart; + + /* Each PUD is 8GB */ + if ((vstart & mask16gb) || + (vend - vstart <= mask16gb)) { + pte_val ^= kern_linear_pte_xor[2]; + pud_val(*pud) = pte_val | _PAGE_PUD_HUGE; + + return vstart + PUD_SIZE; + } + + pte_val ^= kern_linear_pte_xor[3]; + pte_val |= _PAGE_PUD_HUGE; + + vend = vstart + mask16gb + 1UL; + while (vstart < vend) { + pud_val(*pud) = pte_val; + + pte_val += PUD_SIZE; + vstart += PUD_SIZE; + pud++; + } + return vstart; +} + +static bool kernel_can_map_hugepud(unsigned long vstart, unsigned long vend, + bool guard) +{ + if (guard && !(vstart & ~PUD_MASK) && (vend - vstart) >= PUD_SIZE) + return true; + + return false; +} + +static unsigned long __ref kernel_map_hugepmd(unsigned long vstart, + unsigned long vend, + pmd_t *pmd) +{ + const unsigned long mask256mb = (1UL << 28) - 1UL; + const unsigned long mask2gb = (1UL << 31) - 1UL; + u64 pte_val = vstart; + + /* Each PMD is 8MB */ + if ((vstart & mask256mb) || + (vend - vstart <= mask256mb)) { + pte_val ^= kern_linear_pte_xor[0]; + pmd_val(*pmd) = pte_val | _PAGE_PMD_HUGE; + + return vstart + PMD_SIZE; + } + + if ((vstart & mask2gb) || + (vend - vstart <= mask2gb)) { + pte_val ^= kern_linear_pte_xor[1]; + pte_val |= _PAGE_PMD_HUGE; + vend = vstart + mask256mb + 1UL; + } else { + pte_val ^= kern_linear_pte_xor[2]; + pte_val |= _PAGE_PMD_HUGE; + vend = vstart + mask2gb + 1UL; + } + + while (vstart < vend) { + pmd_val(*pmd) = pte_val; + + pte_val += PMD_SIZE; + vstart += PMD_SIZE; + pmd++; + } + + return vstart; +} + +static bool kernel_can_map_hugepmd(unsigned long vstart, unsigned long vend, + bool guard) +{ + if (guard && !(vstart & ~PMD_MASK) && (vend - vstart) >= PMD_SIZE) + return true; + + return false; +} + static unsigned long __ref kernel_map_range(unsigned long pstart, - unsigned long pend, pgprot_t prot) + unsigned long pend, pgprot_t prot, + bool use_huge) { unsigned long vstart = PAGE_OFFSET + pstart; unsigned long vend = PAGE_OFFSET + pend; @@ -1372,24 +1779,62 @@ static unsigned long __ref kernel_map_range(unsigned long pstart, while (vstart < vend) { unsigned long this_end, paddr = __pa(vstart); pgd_t *pgd = pgd_offset_k(vstart); + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte; - pud = pud_offset(pgd, vstart); + if (pgd_none(*pgd)) { + pud_t *new; + + new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, + PAGE_SIZE); + if (!new) + goto err_alloc; + alloc_bytes += PAGE_SIZE; + pgd_populate(&init_mm, pgd, new); + } + + p4d = p4d_offset(pgd, vstart); + if (p4d_none(*p4d)) { + pud_t *new; + + new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, + PAGE_SIZE); + if (!new) + goto err_alloc; + alloc_bytes += PAGE_SIZE; + p4d_populate(&init_mm, p4d, new); + } + + pud = pud_offset(p4d, vstart); if (pud_none(*pud)) { pmd_t *new; - new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); + if (kernel_can_map_hugepud(vstart, vend, use_huge)) { + vstart = kernel_map_hugepud(vstart, vend, pud); + continue; + } + new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, + PAGE_SIZE); + if (!new) + goto err_alloc; alloc_bytes += PAGE_SIZE; pud_populate(&init_mm, pud, new); } pmd = pmd_offset(pud, vstart); - if (!pmd_present(*pmd)) { + if (pmd_none(*pmd)) { pte_t *new; - new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); + if (kernel_can_map_hugepmd(vstart, vend, use_huge)) { + vstart = kernel_map_hugepmd(vstart, vend, pmd); + continue; + } + new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, + PAGE_SIZE); + if (!new) + goto err_alloc; alloc_bytes += PAGE_SIZE; pmd_populate_kernel(&init_mm, pmd, new); } @@ -1409,102 +1854,41 @@ static unsigned long __ref kernel_map_range(unsigned long pstart, } return alloc_bytes; -} - -extern unsigned int kvmap_linear_patch[1]; -#endif /* CONFIG_DEBUG_PAGEALLOC */ - -static void __init kpte_set_val(unsigned long index, unsigned long val) -{ - unsigned long *ptr = kpte_linear_bitmap; - val <<= ((index % (BITS_PER_LONG / 2)) * 2); - ptr += (index / (BITS_PER_LONG / 2)); - - *ptr |= val; +err_alloc: + panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); + return -ENOMEM; } -static const unsigned long kpte_shift_min = 28; /* 256MB */ -static const unsigned long kpte_shift_max = 34; /* 16GB */ -static const unsigned long kpte_shift_incr = 3; - -static unsigned long kpte_mark_using_shift(unsigned long start, unsigned long end, - unsigned long shift) +static void __init flush_all_kernel_tsbs(void) { - unsigned long size = (1UL << shift); - unsigned long mask = (size - 1UL); - unsigned long remains = end - start; - unsigned long val; - - if (remains < size || (start & mask)) - return start; - - /* VAL maps: - * - * shift 28 --> kern_linear_pte_xor index 1 - * shift 31 --> kern_linear_pte_xor index 2 - * shift 34 --> kern_linear_pte_xor index 3 - */ - val = ((shift - kpte_shift_min) / kpte_shift_incr) + 1; - - remains &= ~mask; - if (shift != kpte_shift_max) - remains = size; - - while (remains) { - unsigned long index = start >> kpte_shift_min; + int i; - kpte_set_val(index, val); + for (i = 0; i < KERNEL_TSB_NENTRIES; i++) { + struct tsb *ent = &swapper_tsb[i]; - start += 1UL << kpte_shift_min; - remains -= 1UL << kpte_shift_min; + ent->tag = (1UL << TSB_TAG_INVALID_BIT); } +#ifndef CONFIG_DEBUG_PAGEALLOC + for (i = 0; i < KERNEL_TSB4M_NENTRIES; i++) { + struct tsb *ent = &swapper_4m_tsb[i]; - return start; -} - -static void __init mark_kpte_bitmap(unsigned long start, unsigned long end) -{ - unsigned long smallest_size, smallest_mask; - unsigned long s; - - smallest_size = (1UL << kpte_shift_min); - smallest_mask = (smallest_size - 1UL); - - while (start < end) { - unsigned long orig_start = start; - - for (s = kpte_shift_max; s >= kpte_shift_min; s -= kpte_shift_incr) { - start = kpte_mark_using_shift(start, end, s); - - if (start != orig_start) - break; - } - - if (start == orig_start) - start = (start + smallest_size) & ~smallest_mask; + ent->tag = (1UL << TSB_TAG_INVALID_BIT); } +#endif } -static void __init init_kpte_bitmap(void) -{ - unsigned long i; - - for (i = 0; i < pall_ents; i++) { - unsigned long phys_start, phys_end; - - phys_start = pall[i].phys_addr; - phys_end = phys_start + pall[i].reg_size; - - mark_kpte_bitmap(phys_start, phys_end); - } -} +extern unsigned int kvmap_linear_patch[1]; static void __init kernel_physical_mapping_init(void) { -#ifdef CONFIG_DEBUG_PAGEALLOC unsigned long i, mem_alloced = 0UL; + bool use_huge = true; +#ifdef CONFIG_DEBUG_PAGEALLOC + use_huge = false; +#endif for (i = 0; i < pall_ents; i++) { unsigned long phys_start, phys_end; @@ -1512,7 +1896,7 @@ static void __init kernel_physical_mapping_init(void) phys_end = phys_start + pall[i].reg_size; mem_alloced += kernel_map_range(phys_start, phys_end, - PAGE_KERNEL); + PAGE_KERNEL, use_huge); } printk("Allocated %ld bytes for kernel page tables.\n", @@ -1521,18 +1905,19 @@ static void __init kernel_physical_mapping_init(void) kvmap_linear_patch[0] = 0x01000000; /* nop */ flushi(&kvmap_linear_patch[0]); + flush_all_kernel_tsbs(); + __flush_tlb_all(); -#endif } #ifdef CONFIG_DEBUG_PAGEALLOC -void kernel_map_pages(struct page *page, int numpages, int enable) +void __kernel_map_pages(struct page *page, int numpages, int enable) { unsigned long phys_start = page_to_pfn(page) << PAGE_SHIFT; unsigned long phys_end = phys_start + (numpages * PAGE_SIZE); kernel_map_range(phys_start, phys_end, - (enable ? PAGE_KERNEL : __pgprot(0))); + (enable ? PAGE_KERNEL : __pgprot(0)), false); flush_tsb_kernel_range(PAGE_OFFSET + phys_start, PAGE_OFFSET + phys_end); @@ -1557,6 +1942,91 @@ unsigned long __init find_ecache_flush_span(unsigned long size) return ~0UL; } +unsigned long PAGE_OFFSET; +EXPORT_SYMBOL(PAGE_OFFSET); + +unsigned long VMALLOC_END = 0x0000010000000000UL; +EXPORT_SYMBOL(VMALLOC_END); + +unsigned long sparc64_va_hole_top = 0xfffff80000000000UL; +unsigned long sparc64_va_hole_bottom = 0x0000080000000000UL; + +static void __init setup_page_offset(void) +{ + if (tlb_type == cheetah || tlb_type == cheetah_plus) { + /* Cheetah/Panther support a full 64-bit virtual + * address, so we can use all that our page tables + * support. + */ + sparc64_va_hole_top = 0xfff0000000000000UL; + sparc64_va_hole_bottom = 0x0010000000000000UL; + + max_phys_bits = 42; + } else if (tlb_type == hypervisor) { + switch (sun4v_chip_type) { + case SUN4V_CHIP_NIAGARA1: + case SUN4V_CHIP_NIAGARA2: + /* T1 and T2 support 48-bit virtual addresses. */ + sparc64_va_hole_top = 0xffff800000000000UL; + sparc64_va_hole_bottom = 0x0000800000000000UL; + + max_phys_bits = 39; + break; + case SUN4V_CHIP_NIAGARA3: + /* T3 supports 48-bit virtual addresses. */ + sparc64_va_hole_top = 0xffff800000000000UL; + sparc64_va_hole_bottom = 0x0000800000000000UL; + + max_phys_bits = 43; + break; + case SUN4V_CHIP_NIAGARA4: + case SUN4V_CHIP_NIAGARA5: + case SUN4V_CHIP_SPARC64X: + case SUN4V_CHIP_SPARC_M6: + /* T4 and later support 52-bit virtual addresses. */ + sparc64_va_hole_top = 0xfff8000000000000UL; + sparc64_va_hole_bottom = 0x0008000000000000UL; + max_phys_bits = 47; + break; + case SUN4V_CHIP_SPARC_M7: + case SUN4V_CHIP_SPARC_SN: + /* M7 and later support 52-bit virtual addresses. */ + sparc64_va_hole_top = 0xfff8000000000000UL; + sparc64_va_hole_bottom = 0x0008000000000000UL; + max_phys_bits = 49; + break; + case SUN4V_CHIP_SPARC_M8: + default: + /* M8 and later support 54-bit virtual addresses. + * However, restricting M8 and above VA bits to 53 + * as 4-level page table cannot support more than + * 53 VA bits. + */ + sparc64_va_hole_top = 0xfff0000000000000UL; + sparc64_va_hole_bottom = 0x0010000000000000UL; + max_phys_bits = 51; + break; + } + } + + if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) { + prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n", + max_phys_bits); + prom_halt(); + } + + PAGE_OFFSET = sparc64_va_hole_top; + VMALLOC_END = ((sparc64_va_hole_bottom >> 1) + + (sparc64_va_hole_bottom >> 2)); + + pr_info("MM: PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n", + PAGE_OFFSET, max_phys_bits); + pr_info("MM: VMALLOC [0x%016lx --> 0x%016lx]\n", + VMALLOC_START, VMALLOC_END); + pr_info("MM: VMEMMAP [0x%016lx --> 0x%016lx]\n", + VMEMMAP_BASE, VMEMMAP_BASE << 1); +} + static void __init tsb_phys_patch(void) { struct tsb_ldquad_phys_patch_entry *pquad; @@ -1599,21 +2069,42 @@ static void __init tsb_phys_patch(void) #define NUM_KTSB_DESCR 1 #endif static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR]; -extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; + +/* The swapper TSBs are loaded with a base sequence of: + * + * sethi %uhi(SYMBOL), REG1 + * sethi %hi(SYMBOL), REG2 + * or REG1, %ulo(SYMBOL), REG1 + * or REG2, %lo(SYMBOL), REG2 + * sllx REG1, 32, REG1 + * or REG1, REG2, REG1 + * + * When we use physical addressing for the TSB accesses, we patch the + * first four instructions in the above sequence. + */ static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa) { - pa >>= KTSB_PHYS_SHIFT; + unsigned long high_bits, low_bits; + + high_bits = (pa >> 32) & 0xffffffff; + low_bits = (pa >> 0) & 0xffffffff; while (start < end) { unsigned int *ia = (unsigned int *)(unsigned long)*start; - ia[0] = (ia[0] & ~0x3fffff) | (pa >> 10); + ia[0] = (ia[0] & ~0x3fffff) | (high_bits >> 10); __asm__ __volatile__("flush %0" : : "r" (ia)); - ia[1] = (ia[1] & ~0x3ff) | (pa & 0x3ff); + ia[1] = (ia[1] & ~0x3fffff) | (low_bits >> 10); __asm__ __volatile__("flush %0" : : "r" (ia + 1)); + ia[2] = (ia[2] & ~0x1fff) | (high_bits & 0x3ff); + __asm__ __volatile__("flush %0" : : "r" (ia + 2)); + + ia[3] = (ia[3] & ~0x1fff) | (low_bits & 0x3ff); + __asm__ __volatile__("flush %0" : : "r" (ia + 3)); + start++; } } @@ -1694,7 +2185,7 @@ static void __init sun4v_ktsb_init(void) #endif } -void __cpuinit sun4v_ktsb_register(void) +void sun4v_ktsb_register(void) { unsigned long pa, ret; @@ -1719,11 +2210,26 @@ static void __init sun4u_linear_pte_xor_finalize(void) static void __init sun4v_linear_pte_xor_finalize(void) { + unsigned long pagecv_flag; + + /* Bit 9 of TTE is no longer CV bit on M7 processor and it instead + * enables MCD error. Do not set bit 9 on M7 processor. + */ + switch (sun4v_chip_type) { + case SUN4V_CHIP_SPARC_M7: + case SUN4V_CHIP_SPARC_M8: + case SUN4V_CHIP_SPARC_SN: + pagecv_flag = 0x00; + break; + default: + pagecv_flag = _PAGE_CV_4V; + break; + } #ifndef CONFIG_DEBUG_PAGEALLOC if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) { kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^ - 0xfffff80000000000UL; - kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V | + PAGE_OFFSET; + kern_linear_pte_xor[1] |= (_PAGE_CP_4V | pagecv_flag | _PAGE_P_4V | _PAGE_W_4V); } else { kern_linear_pte_xor[1] = kern_linear_pte_xor[0]; @@ -1731,8 +2237,8 @@ static void __init sun4v_linear_pte_xor_finalize(void) if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) { kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^ - 0xfffff80000000000UL; - kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V | + PAGE_OFFSET; + kern_linear_pte_xor[2] |= (_PAGE_CP_4V | pagecv_flag | _PAGE_P_4V | _PAGE_W_4V); } else { kern_linear_pte_xor[2] = kern_linear_pte_xor[1]; @@ -1740,8 +2246,8 @@ static void __init sun4v_linear_pte_xor_finalize(void) if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) { kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^ - 0xfffff80000000000UL; - kern_linear_pte_xor[3] |= (_PAGE_CP_4V | _PAGE_CV_4V | + PAGE_OFFSET; + kern_linear_pte_xor[3] |= (_PAGE_CP_4V | pagecv_flag | _PAGE_P_4V | _PAGE_W_4V); } else { kern_linear_pte_xor[3] = kern_linear_pte_xor[2]; @@ -1752,22 +2258,39 @@ static void __init sun4v_linear_pte_xor_finalize(void) /* paging_init() sets up the page tables */ static unsigned long last_valid_pfn; -pgd_t swapper_pg_dir[2048]; static void sun4u_pgprot_init(void); static void sun4v_pgprot_init(void); +#define _PAGE_CACHE_4U (_PAGE_CP_4U | _PAGE_CV_4U) +#define _PAGE_CACHE_4V (_PAGE_CP_4V | _PAGE_CV_4V) +#define __DIRTY_BITS_4U (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U) +#define __DIRTY_BITS_4V (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V) +#define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R) +#define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R) + +/* We need to exclude reserved regions. This exclusion will include + * vmlinux and initrd. To be more precise the initrd size could be used to + * compute a new lower limit because it is freed later during initialization. + */ +static void __init reduce_memory(phys_addr_t limit_ram) +{ + limit_ram += memblock_reserved_size(); + memblock_enforce_memory_limit(limit_ram); +} + void __init paging_init(void) { unsigned long end_pfn, shift, phys_base; unsigned long real_end, i; - int node; + + setup_page_offset(); /* These build time checkes make sure that the dcache_dirty_cpu() - * page->flags usage will work. + * folio->flags usage will work. * * When a page gets marked as dcache-dirty, we store the - * cpu number starting at bit 32 in the page->flags. Also, + * cpu number starting at bit 32 in the folio->flags. Also, * functions like clear_dcache_dirty_cpu use the cpu mask * in 13-bit signed-immediate instruction fields. */ @@ -1788,7 +2311,7 @@ void __init paging_init(void) BUILD_BUG_ON(NR_CPUS > 4096); - kern_base = (prom_boot_mapping_phys_low >> 22UL) << 22UL; + kern_base = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB; kern_size = (unsigned long)&_end - (unsigned long)KERNBASE; /* Invalidate both kernel TSBs. */ @@ -1797,6 +2320,27 @@ void __init paging_init(void) memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb)); #endif + /* TTE.cv bit on sparc v9 occupies the same position as TTE.mcde + * bit on M7 processor. This is a conflicting usage of the same + * bit. Enabling TTE.cv on M7 would turn on Memory Corruption + * Detection error on all pages and this will lead to problems + * later. Kernel does not run with MCD enabled and hence rest + * of the required steps to fully configure memory corruption + * detection are not taken. We need to ensure TTE.mcde is not + * set on M7 processor. Compute the value of cacheability + * flag for use later taking this into consideration. + */ + switch (sun4v_chip_type) { + case SUN4V_CHIP_SPARC_M7: + case SUN4V_CHIP_SPARC_M8: + case SUN4V_CHIP_SPARC_SN: + page_cache4v_flag = _PAGE_CP_4V; + break; + default: + page_cache4v_flag = _PAGE_CACHE_4V; + break; + } + if (tlb_type == hypervisor) sun4v_pgprot_init(); else @@ -1834,7 +2378,8 @@ void __init paging_init(void) find_ramdisk(phys_base); - memblock_enforce_memory_limit(cmdline_memory_size); + if (cmdline_memory_size) + reduce_memory(cmdline_memory_size); memblock_allow_resize(); memblock_dump_all(); @@ -1844,7 +2389,7 @@ void __init paging_init(void) shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE); real_end = (unsigned long)_end; - num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << 22); + num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << ILOG2_4MB); printk("Kernel: Using %d locked TLB entries for main kernel image.\n", num_kernel_image_mappings); @@ -1853,16 +2398,10 @@ void __init paging_init(void) */ init_mm.pgd += ((shift) / (sizeof(pgd_t))); - memset(swapper_low_pmd_dir, 0, sizeof(swapper_low_pmd_dir)); + memset(swapper_pg_dir, 0, sizeof(swapper_pg_dir)); - /* Now can init the kernel/bad page tables. */ - pud_set(pud_offset(&swapper_pg_dir[0], 0), - swapper_low_pmd_dir + (shift / sizeof(pgd_t))); - inherit_prom_mappings(); - init_kpte_bitmap(); - /* Ok, we can use our TLB miss and window trap handlers safely. */ setup_tba(); @@ -1913,21 +2452,6 @@ void __init paging_init(void) /* Setup bootmem... */ last_valid_pfn = end_pfn = bootmem_init(phys_base); - /* Once the OF device tree and MDESC have been setup, we know - * the list of possible cpus. Therefore we can allocate the - * IRQ stacks. - */ - for_each_possible_cpu(i) { - node = cpu_to_node(i); - - softirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node), - THREAD_SIZE, - THREAD_SIZE, 0); - hardirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node), - THREAD_SIZE, - THREAD_SIZE, 0); - } - kernel_physical_mapping_init(); { @@ -1937,7 +2461,7 @@ void __init paging_init(void) max_zone_pfns[ZONE_NORMAL] = end_pfn; - free_area_init_nodes(max_zone_pfns); + free_area_init(max_zone_pfns); } printk("Booting Linux...\n"); @@ -1969,73 +2493,9 @@ int page_in_phys_avail(unsigned long paddr) return 0; } -static struct linux_prom64_registers pavail_rescan[MAX_BANKS] __initdata; -static int pavail_rescan_ents __initdata; - -/* Certain OBP calls, such as fetching "available" properties, can - * claim physical memory. So, along with initializing the valid - * address bitmap, what we do here is refetch the physical available - * memory list again, and make sure it provides at least as much - * memory as 'pavail' does. - */ -static void __init setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap) -{ - int i; - - read_obp_memory("available", &pavail_rescan[0], &pavail_rescan_ents); - - for (i = 0; i < pavail_ents; i++) { - unsigned long old_start, old_end; - - old_start = pavail[i].phys_addr; - old_end = old_start + pavail[i].reg_size; - while (old_start < old_end) { - int n; - - for (n = 0; n < pavail_rescan_ents; n++) { - unsigned long new_start, new_end; - - new_start = pavail_rescan[n].phys_addr; - new_end = new_start + - pavail_rescan[n].reg_size; - - if (new_start <= old_start && - new_end >= (old_start + PAGE_SIZE)) { - set_bit(old_start >> 22, bitmap); - goto do_next_page; - } - } - - prom_printf("mem_init: Lost memory in pavail\n"); - prom_printf("mem_init: OLD start[%lx] size[%lx]\n", - pavail[i].phys_addr, - pavail[i].reg_size); - prom_printf("mem_init: NEW start[%lx] size[%lx]\n", - pavail_rescan[i].phys_addr, - pavail_rescan[i].reg_size); - prom_printf("mem_init: Cannot continue, aborting.\n"); - prom_halt(); - - do_next_page: - old_start += PAGE_SIZE; - } - } -} - -static void __init patch_tlb_miss_handler_bitmap(void) -{ - extern unsigned int valid_addr_bitmap_insn[]; - extern unsigned int valid_addr_bitmap_patch[]; - - valid_addr_bitmap_insn[1] = valid_addr_bitmap_patch[1]; - mb(); - valid_addr_bitmap_insn[0] = valid_addr_bitmap_patch[0]; - flushi(&valid_addr_bitmap_insn[0]); -} - static void __init register_page_bootmem_info(void) { -#ifdef CONFIG_NEED_MULTIPLE_NODES +#ifdef CONFIG_NUMA int i; for_each_online_node(i) @@ -2045,22 +2505,13 @@ static void __init register_page_bootmem_info(void) } void __init mem_init(void) { - unsigned long addr, last; - - addr = PAGE_OFFSET + kern_base; - last = PAGE_ALIGN(kern_size) + addr; - while (addr < last) { - set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap); - addr += PAGE_SIZE; - } - - setup_valid_addr_bitmap_from_pavail(sparc64_valid_addr_bitmap); - patch_tlb_miss_handler_bitmap(); - - high_memory = __va(last_valid_pfn << PAGE_SHIFT); - + /* + * Must be done after boot memory is put on freelist, because here we + * might set fields in deferred struct pages that have not yet been + * initialized, and memblock_free_all() initializes all the reserved + * deferred pages for us. + */ register_page_bootmem_info(); - free_all_bootmem(); /* * Set up the zero page, mark it reserved, so that page count @@ -2073,7 +2524,6 @@ void __init mem_init(void) } mark_page_reserved(mem_map_zero); - mem_init_print_info(NULL); if (tlb_type == cheetah || tlb_type == cheetah_plus) cheetah_ecache_flush_init(); @@ -2110,21 +2560,6 @@ void free_initmem(void) } } -#ifdef CONFIG_BLK_DEV_INITRD -void free_initrd_mem(unsigned long start, unsigned long end) -{ - free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM, - "initrd"); -} -#endif - -#define _PAGE_CACHE_4U (_PAGE_CP_4U | _PAGE_CV_4U) -#define _PAGE_CACHE_4V (_PAGE_CP_4V | _PAGE_CV_4V) -#define __DIRTY_BITS_4U (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U) -#define __DIRTY_BITS_4V (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V) -#define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R) -#define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R) - pgprot_t PAGE_KERNEL __read_mostly; EXPORT_SYMBOL(PAGE_KERNEL); @@ -2146,18 +2581,9 @@ unsigned long _PAGE_CACHE __read_mostly; EXPORT_SYMBOL(_PAGE_CACHE); #ifdef CONFIG_SPARSEMEM_VMEMMAP -unsigned long vmemmap_table[VMEMMAP_SIZE]; - -static long __meminitdata addr_start, addr_end; -static int __meminitdata node_start; - int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend, - int node) + int node, struct vmem_altmap *altmap) { - unsigned long phys_start = (vstart - VMEMMAP_BASE); - unsigned long phys_end = (vend - VMEMMAP_BASE); - unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK; - unsigned long end = VMEMMAP_ALIGN(phys_end); unsigned long pte_base; pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U | @@ -2165,52 +2591,49 @@ int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend, _PAGE_P_4U | _PAGE_W_4U); if (tlb_type == hypervisor) pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4V | - _PAGE_CP_4V | _PAGE_CV_4V | - _PAGE_P_4V | _PAGE_W_4V); + page_cache4v_flag | _PAGE_P_4V | _PAGE_W_4V); + + pte_base |= _PAGE_PMD_HUGE; + + vstart = vstart & PMD_MASK; + vend = ALIGN(vend, PMD_SIZE); + for (; vstart < vend; vstart += PMD_SIZE) { + pgd_t *pgd = vmemmap_pgd_populate(vstart, node); + unsigned long pte; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; - for (; addr < end; addr += VMEMMAP_CHUNK) { - unsigned long *vmem_pp = - vmemmap_table + (addr >> VMEMMAP_CHUNK_SHIFT); - void *block; + if (!pgd) + return -ENOMEM; + + p4d = vmemmap_p4d_populate(pgd, vstart, node); + if (!p4d) + return -ENOMEM; + + pud = vmemmap_pud_populate(p4d, vstart, node); + if (!pud) + return -ENOMEM; + + pmd = pmd_offset(pud, vstart); + pte = pmd_val(*pmd); + if (!(pte & _PAGE_VALID)) { + void *block = vmemmap_alloc_block(PMD_SIZE, node); - if (!(*vmem_pp & _PAGE_VALID)) { - block = vmemmap_alloc_block(1UL << 22, node); if (!block) return -ENOMEM; - *vmem_pp = pte_base | __pa(block); - - /* check to see if we have contiguous blocks */ - if (addr_end != addr || node_start != node) { - if (addr_start) - printk(KERN_DEBUG " [%lx-%lx] on node %d\n", - addr_start, addr_end-1, node_start); - addr_start = addr; - node_start = node; - } - addr_end = addr + VMEMMAP_CHUNK; + pmd_val(*pmd) = pte_base | __pa(block); } } - return 0; -} -void __meminit vmemmap_populate_print_last(void) -{ - if (addr_start) { - printk(KERN_DEBUG " [%lx-%lx] on node %d\n", - addr_start, addr_end-1, node_start); - addr_start = 0; - addr_end = 0; - node_start = 0; - } -} - -void vmemmap_free(unsigned long start, unsigned long end) -{ + return 0; } - #endif /* CONFIG_SPARSEMEM_VMEMMAP */ +/* These are actually filled in at boot time by sun4{u,v}_pgprot_init() */ +static pgprot_t protection_map[16] __ro_after_init; + static void prot_init_common(unsigned long page_none, unsigned long page_shared, unsigned long page_copy, @@ -2261,10 +2684,10 @@ static void __init sun4u_pgprot_init(void) __ACCESS_BITS_4U | _PAGE_E_4U); #ifdef CONFIG_DEBUG_PAGEALLOC - kern_linear_pte_xor[0] = _PAGE_VALID ^ 0xfffff80000000000UL; + kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET; #else kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^ - 0xfffff80000000000UL; + PAGE_OFFSET; #endif kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U | _PAGE_W_4U); @@ -2298,23 +2721,23 @@ static void __init sun4v_pgprot_init(void) int i; PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID | - _PAGE_CACHE_4V | _PAGE_P_4V | + page_cache4v_flag | _PAGE_P_4V | __ACCESS_BITS_4V | __DIRTY_BITS_4V | _PAGE_EXEC_4V); PAGE_KERNEL_LOCKED = PAGE_KERNEL; _PAGE_IE = _PAGE_IE_4V; _PAGE_E = _PAGE_E_4V; - _PAGE_CACHE = _PAGE_CACHE_4V; + _PAGE_CACHE = page_cache4v_flag; #ifdef CONFIG_DEBUG_PAGEALLOC - kern_linear_pte_xor[0] = _PAGE_VALID ^ 0xfffff80000000000UL; + kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET; #else kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^ - 0xfffff80000000000UL; + PAGE_OFFSET; #endif - kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V | - _PAGE_P_4V | _PAGE_W_4V); + kern_linear_pte_xor[0] |= (page_cache4v_flag | _PAGE_P_4V | + _PAGE_W_4V); for (i = 1; i < 4; i++) kern_linear_pte_xor[i] = kern_linear_pte_xor[0]; @@ -2327,12 +2750,12 @@ static void __init sun4v_pgprot_init(void) _PAGE_SZ4MB_4V | _PAGE_SZ512K_4V | _PAGE_SZ64K_4V | _PAGE_SZ8K_4V); - page_none = _PAGE_PRESENT_4V | _PAGE_ACCESSED_4V | _PAGE_CACHE_4V; - page_shared = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V | + page_none = _PAGE_PRESENT_4V | _PAGE_ACCESSED_4V | page_cache4v_flag; + page_shared = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag | __ACCESS_BITS_4V | _PAGE_WRITE_4V | _PAGE_EXEC_4V); - page_copy = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V | + page_copy = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag | __ACCESS_BITS_4V | _PAGE_EXEC_4V); - page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V | + page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag | __ACCESS_BITS_4V | _PAGE_EXEC_4V); page_exec_bit = _PAGE_EXEC_4V; @@ -2390,7 +2813,7 @@ static unsigned long kern_large_tte(unsigned long paddr) _PAGE_EXEC_4U | _PAGE_L_4U | _PAGE_W_4U); if (tlb_type == hypervisor) val = (_PAGE_VALID | _PAGE_SZ4MB_4V | - _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_P_4V | + page_cache4v_flag | _PAGE_P_4V | _PAGE_EXEC_4V | _PAGE_W_4V); return val | paddr; @@ -2455,92 +2878,40 @@ void __flush_tlb_all(void) : : "r" (pstate)); } -static pte_t *get_from_cache(struct mm_struct *mm) +static pte_t *__pte_alloc_one(struct mm_struct *mm) { - struct page *page; - pte_t *ret; + struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0); - spin_lock(&mm->page_table_lock); - page = mm->context.pgtable_page; - ret = NULL; - if (page) { - void *p = page_address(page); - - mm->context.pgtable_page = NULL; - - ret = (pte_t *) (p + (PAGE_SIZE / 2)); + if (!ptdesc) + return NULL; + if (!pagetable_pte_ctor(mm, ptdesc)) { + pagetable_free(ptdesc); + return NULL; } - spin_unlock(&mm->page_table_lock); - - return ret; + return ptdesc_address(ptdesc); } -static struct page *__alloc_for_cache(struct mm_struct *mm) +pte_t *pte_alloc_one_kernel(struct mm_struct *mm) { - struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | - __GFP_REPEAT | __GFP_ZERO); - - if (page) { - spin_lock(&mm->page_table_lock); - if (!mm->context.pgtable_page) { - atomic_set(&page->_count, 2); - mm->context.pgtable_page = page; - } - spin_unlock(&mm->page_table_lock); - } - return page; + return __pte_alloc_one(mm); } -pte_t *pte_alloc_one_kernel(struct mm_struct *mm, - unsigned long address) +pgtable_t pte_alloc_one(struct mm_struct *mm) { - struct page *page; - pte_t *pte; - - pte = get_from_cache(mm); - if (pte) - return pte; - - page = __alloc_for_cache(mm); - if (page) - pte = (pte_t *) page_address(page); - - return pte; + return __pte_alloc_one(mm); } -pgtable_t pte_alloc_one(struct mm_struct *mm, - unsigned long address) +static void __pte_free(pgtable_t pte) { - struct page *page; - pte_t *pte; - - pte = get_from_cache(mm); - if (pte) - return pte; - - page = __alloc_for_cache(mm); - if (page) { - pgtable_page_ctor(page); - pte = (pte_t *) page_address(page); - } + struct ptdesc *ptdesc = virt_to_ptdesc(pte); - return pte; + pagetable_dtor(ptdesc); + pagetable_free(ptdesc); } void pte_free_kernel(struct mm_struct *mm, pte_t *pte) { - struct page *page = virt_to_page(pte); - if (put_page_testzero(page)) - free_hot_cold_page(page, 0); -} - -static void __pte_free(pgtable_t pte) -{ - struct page *page = virt_to_page(pte); - if (put_page_testzero(page)) { - pgtable_page_dtor(page); - free_hot_cold_page(page, 0); - } + __pte_free(pte); } void pte_free(struct mm_struct *mm, pgtable_t pte) @@ -2557,92 +2928,20 @@ void pgtable_free(void *table, bool is_page) } #ifdef CONFIG_TRANSPARENT_HUGEPAGE -static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot, bool for_modify) +static void pte_free_now(struct rcu_head *head) { - if (pgprot_val(pgprot) & _PAGE_VALID) - pmd_val(pmd) |= PMD_HUGE_PRESENT; - if (tlb_type == hypervisor) { - if (pgprot_val(pgprot) & _PAGE_WRITE_4V) - pmd_val(pmd) |= PMD_HUGE_WRITE; - if (pgprot_val(pgprot) & _PAGE_EXEC_4V) - pmd_val(pmd) |= PMD_HUGE_EXEC; - - if (!for_modify) { - if (pgprot_val(pgprot) & _PAGE_ACCESSED_4V) - pmd_val(pmd) |= PMD_HUGE_ACCESSED; - if (pgprot_val(pgprot) & _PAGE_MODIFIED_4V) - pmd_val(pmd) |= PMD_HUGE_DIRTY; - } - } else { - if (pgprot_val(pgprot) & _PAGE_WRITE_4U) - pmd_val(pmd) |= PMD_HUGE_WRITE; - if (pgprot_val(pgprot) & _PAGE_EXEC_4U) - pmd_val(pmd) |= PMD_HUGE_EXEC; - - if (!for_modify) { - if (pgprot_val(pgprot) & _PAGE_ACCESSED_4U) - pmd_val(pmd) |= PMD_HUGE_ACCESSED; - if (pgprot_val(pgprot) & _PAGE_MODIFIED_4U) - pmd_val(pmd) |= PMD_HUGE_DIRTY; - } - } - - return pmd; -} - -pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) -{ - pmd_t pmd; - - pmd_val(pmd) = (page_nr << ((PAGE_SHIFT - PMD_PADDR_SHIFT))); - pmd_val(pmd) |= PMD_ISHUGE; - pmd = pmd_set_protbits(pmd, pgprot, false); - return pmd; -} + struct page *page; -pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) -{ - pmd_val(pmd) &= ~(PMD_HUGE_PRESENT | - PMD_HUGE_WRITE | - PMD_HUGE_EXEC); - pmd = pmd_set_protbits(pmd, newprot, true); - return pmd; + page = container_of(head, struct page, rcu_head); + __pte_free((pgtable_t)page_address(page)); } -pgprot_t pmd_pgprot(pmd_t entry) +void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable) { - unsigned long pte = 0; - - if (pmd_val(entry) & PMD_HUGE_PRESENT) - pte |= _PAGE_VALID; - - if (tlb_type == hypervisor) { - if (pmd_val(entry) & PMD_HUGE_PRESENT) - pte |= _PAGE_PRESENT_4V; - if (pmd_val(entry) & PMD_HUGE_EXEC) - pte |= _PAGE_EXEC_4V; - if (pmd_val(entry) & PMD_HUGE_WRITE) - pte |= _PAGE_W_4V; - if (pmd_val(entry) & PMD_HUGE_ACCESSED) - pte |= _PAGE_ACCESSED_4V; - if (pmd_val(entry) & PMD_HUGE_DIRTY) - pte |= _PAGE_MODIFIED_4V; - pte |= _PAGE_CP_4V|_PAGE_CV_4V; - } else { - if (pmd_val(entry) & PMD_HUGE_PRESENT) - pte |= _PAGE_PRESENT_4U; - if (pmd_val(entry) & PMD_HUGE_EXEC) - pte |= _PAGE_EXEC_4U; - if (pmd_val(entry) & PMD_HUGE_WRITE) - pte |= _PAGE_W_4U; - if (pmd_val(entry) & PMD_HUGE_ACCESSED) - pte |= _PAGE_ACCESSED_4U; - if (pmd_val(entry) & PMD_HUGE_DIRTY) - pte |= _PAGE_MODIFIED_4U; - pte |= _PAGE_CP_4U|_PAGE_CV_4U; - } + struct page *page; - return __pgprot(pte); + page = virt_to_page(pgtable); + call_rcu(&page->rcu_head, pte_free_now); } void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, @@ -2651,30 +2950,25 @@ void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, unsigned long pte, flags; struct mm_struct *mm; pmd_t entry = *pmd; - pgprot_t prot; - if (!pmd_large(entry) || !pmd_young(entry)) + if (!pmd_leaf(entry) || !pmd_young(entry)) return; - pte = (pmd_val(entry) & ~PMD_HUGE_PROTBITS); - pte <<= PMD_PADDR_SHIFT; - pte |= _PAGE_VALID; + pte = pmd_val(entry); - prot = pmd_pgprot(entry); - - if (tlb_type == hypervisor) - pgprot_val(prot) |= _PAGE_SZHUGE_4V; - else - pgprot_val(prot) |= _PAGE_SZHUGE_4U; + /* Don't insert a non-valid PMD into the TSB, we'll deadlock. */ + if (!(pte & _PAGE_VALID)) + return; - pte |= pgprot_val(prot); + /* We are fabricating 8MB pages using 4MB real hw pages. */ + pte |= (addr & (1UL << REAL_HPAGE_SHIFT)); mm = vma->vm_mm; spin_lock_irqsave(&mm->context.lock, flags); if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) - __update_mmu_tsb_insert(mm, MM_TSB_HUGE, HPAGE_SHIFT, + __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT, addr, pte); spin_unlock_irqrestore(&mm->context.lock, flags); @@ -2695,7 +2989,7 @@ void hugetlb_setup(struct pt_regs *regs) struct mm_struct *mm = current->mm; struct tsb_config *tp; - if (in_atomic() || !mm) { + if (faulthandler_disabled() || !mm) { const struct exception_table_entry *entry; entry = search_exception_tables(regs->tpc); @@ -2719,9 +3013,10 @@ void hugetlb_setup(struct pt_regs *regs) * the Data-TLB for huge pages. */ if (tlb_type == cheetah_plus) { + bool need_context_reload = false; unsigned long ctx; - spin_lock(&ctx_alloc_lock); + spin_lock_irq(&ctx_alloc_lock); ctx = mm->context.sparc64_ctx_val; ctx &= ~CTX_PGSZ_MASK; ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT; @@ -2740,9 +3035,180 @@ void hugetlb_setup(struct pt_regs *regs) * also executing in this address space. */ mm->context.sparc64_ctx_val = ctx; + need_context_reload = true; + } + spin_unlock_irq(&ctx_alloc_lock); + + if (need_context_reload) on_each_cpu(context_reload, mm, 0); + } +} +#endif + +static struct resource code_resource = { + .name = "Kernel code", + .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM +}; + +static struct resource data_resource = { + .name = "Kernel data", + .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM +}; + +static struct resource bss_resource = { + .name = "Kernel bss", + .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM +}; + +static inline resource_size_t compute_kern_paddr(void *addr) +{ + return (resource_size_t) (addr - KERNBASE + kern_base); +} + +static void __init kernel_lds_init(void) +{ + code_resource.start = compute_kern_paddr(_text); + code_resource.end = compute_kern_paddr(_etext - 1); + data_resource.start = compute_kern_paddr(_etext); + data_resource.end = compute_kern_paddr(_edata - 1); + bss_resource.start = compute_kern_paddr(__bss_start); + bss_resource.end = compute_kern_paddr(_end - 1); +} + +static int __init report_memory(void) +{ + int i; + struct resource *res; + + kernel_lds_init(); + + for (i = 0; i < pavail_ents; i++) { + res = kzalloc(sizeof(struct resource), GFP_KERNEL); + + if (!res) { + pr_warn("Failed to allocate source.\n"); + break; + } + + res->name = "System RAM"; + res->start = pavail[i].phys_addr; + res->end = pavail[i].phys_addr + pavail[i].reg_size - 1; + res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM; + + if (insert_resource(&iomem_resource, res) < 0) { + pr_warn("Resource insertion failed.\n"); + break; } - spin_unlock(&ctx_alloc_lock); + + insert_resource(res, &code_resource); + insert_resource(res, &data_resource); + insert_resource(res, &bss_resource); } + + return 0; } +arch_initcall(report_memory); + +#ifdef CONFIG_SMP +#define do_flush_tlb_kernel_range smp_flush_tlb_kernel_range +#else +#define do_flush_tlb_kernel_range __flush_tlb_kernel_range #endif + +void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + if (start < HI_OBP_ADDRESS && end > LOW_OBP_ADDRESS) { + if (start < LOW_OBP_ADDRESS) { + flush_tsb_kernel_range(start, LOW_OBP_ADDRESS); + do_flush_tlb_kernel_range(start, LOW_OBP_ADDRESS); + } + if (end > HI_OBP_ADDRESS) { + flush_tsb_kernel_range(HI_OBP_ADDRESS, end); + do_flush_tlb_kernel_range(HI_OBP_ADDRESS, end); + } + } else { + flush_tsb_kernel_range(start, end); + do_flush_tlb_kernel_range(start, end); + } +} + +void copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma) +{ + char *vfrom, *vto; + + vfrom = kmap_atomic(from); + vto = kmap_atomic(to); + copy_user_page(vto, vfrom, vaddr, to); + kunmap_atomic(vto); + kunmap_atomic(vfrom); + + /* If this page has ADI enabled, copy over any ADI tags + * as well + */ + if (vma->vm_flags & VM_SPARC_ADI) { + unsigned long pfrom, pto, i, adi_tag; + + pfrom = page_to_phys(from); + pto = page_to_phys(to); + + for (i = pfrom; i < (pfrom + PAGE_SIZE); i += adi_blksize()) { + asm volatile("ldxa [%1] %2, %0\n\t" + : "=r" (adi_tag) + : "r" (i), "i" (ASI_MCD_REAL)); + asm volatile("stxa %0, [%1] %2\n\t" + : + : "r" (adi_tag), "r" (pto), + "i" (ASI_MCD_REAL)); + pto += adi_blksize(); + } + asm volatile("membar #Sync\n\t"); + } +} +EXPORT_SYMBOL(copy_user_highpage); + +void copy_highpage(struct page *to, struct page *from) +{ + char *vfrom, *vto; + + vfrom = kmap_atomic(from); + vto = kmap_atomic(to); + copy_page(vto, vfrom); + kunmap_atomic(vto); + kunmap_atomic(vfrom); + + /* If this platform is ADI enabled, copy any ADI tags + * as well + */ + if (adi_capable()) { + unsigned long pfrom, pto, i, adi_tag; + + pfrom = page_to_phys(from); + pto = page_to_phys(to); + + for (i = pfrom; i < (pfrom + PAGE_SIZE); i += adi_blksize()) { + asm volatile("ldxa [%1] %2, %0\n\t" + : "=r" (adi_tag) + : "r" (i), "i" (ASI_MCD_REAL)); + asm volatile("stxa %0, [%1] %2\n\t" + : + : "r" (adi_tag), "r" (pto), + "i" (ASI_MCD_REAL)); + pto += adi_blksize(); + } + asm volatile("membar #Sync\n\t"); + } +} +EXPORT_SYMBOL(copy_highpage); + +pgprot_t vm_get_page_prot(vm_flags_t vm_flags) +{ + unsigned long prot = pgprot_val(protection_map[vm_flags & + (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]); + + if (vm_flags & VM_SPARC_ADI) + prot |= _PAGE_MCD_4V; + + return __pgprot(prot); +} +EXPORT_SYMBOL(vm_get_page_prot); diff --git a/arch/sparc/mm/init_64.h b/arch/sparc/mm/init_64.h index 0661aa606dec..d920a75b5f14 100644 --- a/arch/sparc/mm/init_64.h +++ b/arch/sparc/mm/init_64.h @@ -1,25 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _SPARC64_MM_INIT_H #define _SPARC64_MM_INIT_H +#include <asm/page.h> + /* Most of the symbols in this file are defined in init.c and * marked non-static so that assembler code can get at them. */ -#define MAX_PHYS_ADDRESS (1UL << 41UL) -#define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL) -#define KPTE_BITMAP_BYTES \ - ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 4) -#define VALID_ADDR_BITMAP_CHUNK_SZ (4UL * 1024UL * 1024UL) -#define VALID_ADDR_BITMAP_BYTES \ - ((MAX_PHYS_ADDRESS / VALID_ADDR_BITMAP_CHUNK_SZ) / 8) +#define MAX_PHYS_ADDRESS (1UL << MAX_PHYS_ADDRESS_BITS) extern unsigned long kern_linear_pte_xor[4]; -extern unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)]; extern unsigned int sparc64_highest_unlocked_tlb_ent; extern unsigned long sparc64_kern_pri_context; extern unsigned long sparc64_kern_pri_nuc_bits; extern unsigned long sparc64_kern_sec_context; -extern void mmu_info(struct seq_file *m); +void mmu_info(struct seq_file *m); struct linux_prom_translation { unsigned long virt; @@ -34,17 +30,6 @@ extern unsigned int prom_trans_ents; /* Exported for SMP bootup purposes. */ extern unsigned long kern_locked_tte_data; -extern void prom_world(int enter); - -#ifdef CONFIG_SPARSEMEM_VMEMMAP -#define VMEMMAP_CHUNK_SHIFT 22 -#define VMEMMAP_CHUNK (1UL << VMEMMAP_CHUNK_SHIFT) -#define VMEMMAP_CHUNK_MASK ~(VMEMMAP_CHUNK - 1UL) -#define VMEMMAP_ALIGN(x) (((x)+VMEMMAP_CHUNK-1UL)&VMEMMAP_CHUNK_MASK) - -#define VMEMMAP_SIZE ((((1UL << MAX_PHYSADDR_BITS) >> PAGE_SHIFT) * \ - sizeof(struct page)) >> VMEMMAP_CHUNK_SHIFT) -extern unsigned long vmemmap_table[VMEMMAP_SIZE]; -#endif +void prom_world(int enter); #endif /* _SPARC64_MM_INIT_H */ diff --git a/arch/sparc/mm/io-unit.c b/arch/sparc/mm/io-unit.c index eb99862e9654..d409cb450de4 100644 --- a/arch/sparc/mm/io-unit.c +++ b/arch/sparc/mm/io-unit.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * io-unit.c: IO-UNIT specific routines for memory management. * @@ -9,14 +10,12 @@ #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/mm.h> -#include <linux/highmem.h> /* pte_offset_map => kmap_atomic */ #include <linux/bitops.h> -#include <linux/scatterlist.h> +#include <linux/dma-map-ops.h> #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> #include <asm/io.h> #include <asm/io-unit.h> #include <asm/mxcc.h> @@ -25,6 +24,8 @@ #include <asm/dma.h> #include <asm/oplib.h> +#include "mm_32.h" + /* #define IOUNIT_DEBUG */ #ifdef IOUNIT_DEBUG #define IOD(x) printk(x) @@ -35,10 +36,13 @@ #define IOPERM (IOUPTE_CACHE | IOUPTE_WRITE | IOUPTE_VALID) #define MKIOPTE(phys) __iopte((((phys)>>4) & IOUPTE_PAGE) | IOPERM) +static const struct dma_map_ops iounit_dma_ops; + static void __init iounit_iommu_init(struct platform_device *op) { struct iounit_struct *iounit; - iopte_t *xpt, *xptend; + iopte_t __iomem *xpt; + iopte_t __iomem *xptend; iounit = kzalloc(sizeof(struct iounit_struct), GFP_ATOMIC); if (!iounit) { @@ -62,10 +66,12 @@ static void __init iounit_iommu_init(struct platform_device *op) op->dev.archdata.iommu = iounit; iounit->page_table = xpt; spin_lock_init(&iounit->lock); - - for (xptend = iounit->page_table + (16 * PAGE_SIZE) / sizeof(iopte_t); - xpt < xptend;) - iopte_val(*xpt++) = 0; + + xptend = iounit->page_table + (16 * PAGE_SIZE) / sizeof(iopte_t); + for (; xpt < xptend; xpt++) + sbus_writel(0, xpt); + + op->dev.dma_ops = &iounit_dma_ops; } static int __init iounit_init(void) @@ -88,13 +94,14 @@ static int __init iounit_init(void) subsys_initcall(iounit_init); /* One has to hold iounit->lock to call this */ -static unsigned long iounit_get_area(struct iounit_struct *iounit, unsigned long vaddr, int size) +static dma_addr_t iounit_get_area(struct iounit_struct *iounit, + phys_addr_t phys, int size) { int i, j, k, npages; unsigned long rotor, scan, limit; iopte_t iopte; - npages = ((vaddr & ~PAGE_MASK) + size + (PAGE_SIZE-1)) >> PAGE_SHIFT; + npages = (offset_in_page(phys) + size + (PAGE_SIZE - 1)) >> PAGE_SHIFT; /* A tiny bit of magic ingredience :) */ switch (npages) { @@ -103,7 +110,7 @@ static unsigned long iounit_get_area(struct iounit_struct *iounit, unsigned long default: i = 0x0213; break; } - IOD(("iounit_get_area(%08lx,%d[%d])=", vaddr, size, npages)); + IOD(("%s(%pa,%d[%d])=", __func__, &phys, size, npages)); next: j = (i & 15); rotor = iounit->rotor[j - 1]; @@ -118,7 +125,8 @@ nexti: scan = find_next_zero_bit(iounit->bmap, limit, scan); } i >>= 4; if (!(i & 15)) - panic("iounit_get_area: Couldn't find free iopte slots for (%08lx,%d)\n", vaddr, size); + panic("iounit_get_area: Couldn't find free iopte slots for (%pa,%d)\n", + &phys, size); goto next; } for (k = 1, scan++; k < npages; k++) @@ -126,44 +134,54 @@ nexti: scan = find_next_zero_bit(iounit->bmap, limit, scan); goto nexti; iounit->rotor[j - 1] = (scan < limit) ? scan : iounit->limit[j - 1]; scan -= npages; - iopte = MKIOPTE(__pa(vaddr & PAGE_MASK)); - vaddr = IOUNIT_DMA_BASE + (scan << PAGE_SHIFT) + (vaddr & ~PAGE_MASK); + iopte = MKIOPTE(phys & PAGE_MASK); + phys = IOUNIT_DMA_BASE + (scan << PAGE_SHIFT) + offset_in_page(phys); for (k = 0; k < npages; k++, iopte = __iopte(iopte_val(iopte) + 0x100), scan++) { set_bit(scan, iounit->bmap); - iounit->page_table[scan] = iopte; + sbus_writel(iopte_val(iopte), &iounit->page_table[scan]); } - IOD(("%08lx\n", vaddr)); - return vaddr; + IOD(("%pa\n", &phys)); + return phys; } -static __u32 iounit_get_scsi_one(struct device *dev, char *vaddr, unsigned long len) +static dma_addr_t iounit_map_phys(struct device *dev, phys_addr_t phys, + size_t len, enum dma_data_direction dir, unsigned long attrs) { struct iounit_struct *iounit = dev->archdata.iommu; - unsigned long ret, flags; + unsigned long flags; + dma_addr_t ret; + /* XXX So what is maxphys for us and how do drivers know it? */ + if (!len || len > 256 * 1024) + return DMA_MAPPING_ERROR; + spin_lock_irqsave(&iounit->lock, flags); - ret = iounit_get_area(iounit, (unsigned long)vaddr, len); + ret = iounit_get_area(iounit, phys, len); spin_unlock_irqrestore(&iounit->lock, flags); return ret; } -static void iounit_get_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz) +static int iounit_map_sg(struct device *dev, struct scatterlist *sgl, int nents, + enum dma_data_direction dir, unsigned long attrs) { struct iounit_struct *iounit = dev->archdata.iommu; + struct scatterlist *sg; unsigned long flags; + int i; /* FIXME: Cache some resolved pages - often several sg entries are to the same page */ spin_lock_irqsave(&iounit->lock, flags); - while (sz != 0) { - --sz; - sg->dma_address = iounit_get_area(iounit, (unsigned long) sg_virt(sg), sg->length); + for_each_sg(sgl, sg, nents, i) { + sg->dma_address = + iounit_get_area(iounit, sg_phys(sg), sg->length); sg->dma_length = sg->length; - sg = sg_next(sg); } spin_unlock_irqrestore(&iounit->lock, flags); + return nents; } -static void iounit_release_scsi_one(struct device *dev, __u32 vaddr, unsigned long len) +static void iounit_unmap_phys(struct device *dev, dma_addr_t vaddr, size_t len, + enum dma_data_direction dir, unsigned long attrs) { struct iounit_struct *iounit = dev->archdata.iommu; unsigned long flags; @@ -177,55 +195,66 @@ static void iounit_release_scsi_one(struct device *dev, __u32 vaddr, unsigned lo spin_unlock_irqrestore(&iounit->lock, flags); } -static void iounit_release_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz) +static void iounit_unmap_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) { struct iounit_struct *iounit = dev->archdata.iommu; - unsigned long flags; - unsigned long vaddr, len; + unsigned long flags, vaddr, len; + struct scatterlist *sg; + int i; spin_lock_irqsave(&iounit->lock, flags); - while (sz != 0) { - --sz; + for_each_sg(sgl, sg, nents, i) { len = ((sg->dma_address & ~PAGE_MASK) + sg->length + (PAGE_SIZE-1)) >> PAGE_SHIFT; vaddr = (sg->dma_address - IOUNIT_DMA_BASE) >> PAGE_SHIFT; IOD(("iounit_release %08lx-%08lx\n", (long)vaddr, (long)len+vaddr)); for (len += vaddr; vaddr < len; vaddr++) clear_bit(vaddr, iounit->bmap); - sg = sg_next(sg); } spin_unlock_irqrestore(&iounit->lock, flags); } #ifdef CONFIG_SBUS -static int iounit_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va, unsigned long addr, int len) +static void *iounit_alloc(struct device *dev, size_t len, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { struct iounit_struct *iounit = dev->archdata.iommu; - unsigned long page, end; + unsigned long va, addr, page, end, ret; pgprot_t dvma_prot; - iopte_t *iopte; + iopte_t __iomem *iopte; - *pba = addr; + /* XXX So what is maxphys for us and how do drivers know it? */ + if (!len || len > 256 * 1024) + return NULL; + + len = PAGE_ALIGN(len); + va = __get_free_pages(gfp | __GFP_ZERO, get_order(len)); + if (!va) + return NULL; + + addr = ret = sparc_dma_alloc_resource(dev, len); + if (!addr) + goto out_free_pages; + *dma_handle = addr; dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV); end = PAGE_ALIGN((addr + len)); while(addr < end) { page = va; { - pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; long i; - pgdp = pgd_offset(&init_mm, addr); - pmdp = pmd_offset(pgdp, addr); - ptep = pte_offset_map(pmdp, addr); + pmdp = pmd_off_k(addr); + ptep = pte_offset_kernel(pmdp, addr); set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot)); - + i = ((addr - IOUNIT_DMA_BASE) >> PAGE_SHIFT); - iopte = (iopte_t *)(iounit->page_table + i); - *iopte = MKIOPTE(__pa(page)); + iopte = iounit->page_table + i; + sbus_writel(iopte_val(MKIOPTE(__pa(page))), iopte); } addr += PAGE_SIZE; va += PAGE_SIZE; @@ -233,27 +262,27 @@ static int iounit_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned lon flush_cache_all(); flush_tlb_all(); - return 0; + return (void *)ret; + +out_free_pages: + free_pages(va, get_order(len)); + return NULL; } -static void iounit_unmap_dma_area(struct device *dev, unsigned long addr, int len) +static void iounit_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_addr, unsigned long attrs) { /* XXX Somebody please fill this in */ } #endif -static const struct sparc32_dma_ops iounit_dma_ops = { - .get_scsi_one = iounit_get_scsi_one, - .get_scsi_sgl = iounit_get_scsi_sgl, - .release_scsi_one = iounit_release_scsi_one, - .release_scsi_sgl = iounit_release_scsi_sgl, +static const struct dma_map_ops iounit_dma_ops = { #ifdef CONFIG_SBUS - .map_dma_area = iounit_map_dma_area, - .unmap_dma_area = iounit_unmap_dma_area, + .alloc = iounit_alloc, + .free = iounit_free, #endif + .map_phys = iounit_map_phys, + .unmap_phys = iounit_unmap_phys, + .map_sg = iounit_map_sg, + .unmap_sg = iounit_unmap_sg, }; - -void __init ld_mmu_iounit(void) -{ - sparc32_dma_ops = &iounit_dma_ops; -} diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c index 28f96f27c768..f48adf62724a 100644 --- a/arch/sparc/mm/iommu.c +++ b/arch/sparc/mm/iommu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * iommu.c: IOMMU specific routines for memory management. * @@ -6,18 +7,16 @@ * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ - + #include <linux/kernel.h> #include <linux/init.h> #include <linux/mm.h> #include <linux/slab.h> -#include <linux/highmem.h> /* pte_offset_map => kmap_atomic */ -#include <linux/scatterlist.h> +#include <linux/dma-map-ops.h> #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> #include <asm/io.h> #include <asm/mxcc.h> #include <asm/mbus.h> @@ -27,6 +26,8 @@ #include <asm/iommu.h> #include <asm/dma.h> +#include "mm_32.h" + /* * This can be sized dynamically, but we will do this * only when we have a guidance about actual I/O pressures. @@ -37,9 +38,6 @@ #define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */ #define IOMMU_ORDER 6 /* 4096 * (1<<6) */ -/* srmmu.c */ -extern int viking_mxcc_present; -extern int flush_page_for_dma_global; static int viking_flush; /* viking.S */ extern void viking_flush_page(unsigned long page); @@ -54,11 +52,16 @@ static pgprot_t dvma_prot; /* Consistent mapping pte flags */ #define IOPERM (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID) #define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ) +static const struct dma_map_ops sbus_iommu_dma_gflush_ops; +static const struct dma_map_ops sbus_iommu_dma_pflush_ops; + static void __init sbus_iommu_init(struct platform_device *op) { struct iommu_struct *iommu; unsigned int impl, vers; unsigned long *bitmap; + unsigned long control; + unsigned long base; unsigned long tmp; iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL); @@ -73,12 +76,14 @@ static void __init sbus_iommu_init(struct platform_device *op) prom_printf("Cannot map IOMMU registers\n"); prom_halt(); } - impl = (iommu->regs->control & IOMMU_CTRL_IMPL) >> 28; - vers = (iommu->regs->control & IOMMU_CTRL_VERS) >> 24; - tmp = iommu->regs->control; - tmp &= ~(IOMMU_CTRL_RNGE); - tmp |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB); - iommu->regs->control = tmp; + + control = sbus_readl(&iommu->regs->control); + impl = (control & IOMMU_CTRL_IMPL) >> 28; + vers = (control & IOMMU_CTRL_VERS) >> 24; + control &= ~(IOMMU_CTRL_RNGE); + control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB); + sbus_writel(control, &iommu->regs->control); + iommu_invalidate(iommu->regs); iommu->start = IOMMU_START; iommu->end = 0xffffffff; @@ -100,7 +105,9 @@ static void __init sbus_iommu_init(struct platform_device *op) memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t)); flush_cache_all(); flush_tlb_all(); - iommu->regs->base = __pa((unsigned long) iommu->page_table) >> 4; + + base = __pa((unsigned long)iommu->page_table) >> 4; + sbus_writel(base, &iommu->regs->base); iommu_invalidate(iommu->regs); bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL); @@ -123,6 +130,11 @@ static void __init sbus_iommu_init(struct platform_device *op) (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES); op->dev.archdata.iommu = iommu; + + if (flush_page_for_dma_global) + op->dev.dma_ops = &sbus_iommu_dma_gflush_ops; + else + op->dev.dma_ops = &sbus_iommu_dma_pflush_ops; } static int __init iommu_init(void) @@ -169,16 +181,39 @@ static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte) } } -static u32 iommu_get_one(struct device *dev, struct page *page, int npages) +static dma_addr_t __sbus_iommu_map_phys(struct device *dev, phys_addr_t paddr, + size_t len, bool per_page_flush, unsigned long attrs) { struct iommu_struct *iommu = dev->archdata.iommu; - int ioptex; - iopte_t *iopte, *iopte0; + unsigned long off = offset_in_page(paddr); + unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT; + unsigned long pfn = __phys_to_pfn(paddr); unsigned int busa, busa0; - int i; + iopte_t *iopte, *iopte0; + int ioptex, i; + + if (unlikely(attrs & DMA_ATTR_MMIO)) + return DMA_MAPPING_ERROR; + + /* XXX So what is maxphys for us and how do drivers know it? */ + if (!len || len > 256 * 1024) + return DMA_MAPPING_ERROR; + + /* + * We expect unmapped highmem pages to be not in the cache. + * XXX Is this a good assumption? + * XXX What if someone else unmaps it here and races us? + */ + if (per_page_flush && !PhysHighMem(paddr)) { + unsigned long vaddr, p; + + vaddr = (unsigned long)phys_to_virt(paddr); + for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE) + flush_page_for_dma(p); + } /* page color = pfn of page */ - ioptex = bit_map_string_get(&iommu->usemap, npages, page_to_pfn(page)); + ioptex = bit_map_string_get(&iommu->usemap, npages, pfn); if (ioptex < 0) panic("iommu out"); busa0 = iommu->start + (ioptex << PAGE_SHIFT); @@ -187,102 +222,74 @@ static u32 iommu_get_one(struct device *dev, struct page *page, int npages) busa = busa0; iopte = iopte0; for (i = 0; i < npages; i++) { - iopte_val(*iopte) = MKIOPTE(page_to_pfn(page), IOPERM); + iopte_val(*iopte) = MKIOPTE(pfn, IOPERM); iommu_invalidate_page(iommu->regs, busa); busa += PAGE_SIZE; iopte++; - page++; + pfn++; } iommu_flush_iotlb(iopte0, npages); - - return busa0; + return busa0 + off; } -static u32 iommu_get_scsi_one(struct device *dev, char *vaddr, unsigned int len) +static dma_addr_t sbus_iommu_map_phys_gflush(struct device *dev, + phys_addr_t phys, size_t len, enum dma_data_direction dir, + unsigned long attrs) { - unsigned long off; - int npages; - struct page *page; - u32 busa; - - off = (unsigned long)vaddr & ~PAGE_MASK; - npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT; - page = virt_to_page((unsigned long)vaddr & PAGE_MASK); - busa = iommu_get_one(dev, page, npages); - return busa + off; + flush_page_for_dma(0); + return __sbus_iommu_map_phys(dev, phys, len, false, attrs); } -static __u32 iommu_get_scsi_one_gflush(struct device *dev, char *vaddr, unsigned long len) +static dma_addr_t sbus_iommu_map_phys_pflush(struct device *dev, + phys_addr_t phys, size_t len, enum dma_data_direction dir, + unsigned long attrs) { - flush_page_for_dma(0); - return iommu_get_scsi_one(dev, vaddr, len); + return __sbus_iommu_map_phys(dev, phys, len, true, attrs); } -static __u32 iommu_get_scsi_one_pflush(struct device *dev, char *vaddr, unsigned long len) +static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs, + bool per_page_flush) { - unsigned long page = ((unsigned long) vaddr) & PAGE_MASK; - - while(page < ((unsigned long)(vaddr + len))) { - flush_page_for_dma(page); - page += PAGE_SIZE; + struct scatterlist *sg; + int j; + + for_each_sg(sgl, sg, nents, j) { + sg->dma_address = __sbus_iommu_map_phys(dev, sg_phys(sg), + sg->length, per_page_flush, attrs); + if (sg->dma_address == DMA_MAPPING_ERROR) + return -EIO; + sg->dma_length = sg->length; } - return iommu_get_scsi_one(dev, vaddr, len); + + return nents; } -static void iommu_get_scsi_sgl_gflush(struct device *dev, struct scatterlist *sg, int sz) +static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) { - int n; - flush_page_for_dma(0); - while (sz != 0) { - --sz; - n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT; - sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset; - sg->dma_length = sg->length; - sg = sg_next(sg); - } + return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false); } -static void iommu_get_scsi_sgl_pflush(struct device *dev, struct scatterlist *sg, int sz) +static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) { - unsigned long page, oldpage = 0; - int n, i; - - while(sz != 0) { - --sz; - - n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT; - - /* - * We expect unmapped highmem pages to be not in the cache. - * XXX Is this a good assumption? - * XXX What if someone else unmaps it here and races us? - */ - if ((page = (unsigned long) page_address(sg_page(sg))) != 0) { - for (i = 0; i < n; i++) { - if (page != oldpage) { /* Already flushed? */ - flush_page_for_dma(page); - oldpage = page; - } - page += PAGE_SIZE; - } - } - - sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset; - sg->dma_length = sg->length; - sg = sg_next(sg); - } + return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true); } -static void iommu_release_one(struct device *dev, u32 busa, int npages) +static void sbus_iommu_unmap_phys(struct device *dev, dma_addr_t dma_addr, + size_t len, enum dma_data_direction dir, unsigned long attrs) { struct iommu_struct *iommu = dev->archdata.iommu; - int ioptex; - int i; + unsigned int busa = dma_addr & PAGE_MASK; + unsigned long off = dma_addr & ~PAGE_MASK; + unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT; + unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT; + unsigned int i; BUG_ON(busa < iommu->start); - ioptex = (busa - iommu->start) >> PAGE_SHIFT; for (i = 0; i < npages; i++) { iopte_val(iommu->page_table[ioptex + i]) = 0; iommu_invalidate_page(iommu->regs, busa); @@ -291,40 +298,42 @@ static void iommu_release_one(struct device *dev, u32 busa, int npages) bit_map_clear(&iommu->usemap, ioptex, npages); } -static void iommu_release_scsi_one(struct device *dev, __u32 vaddr, unsigned long len) +static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) { - unsigned long off; - int npages; - - off = vaddr & ~PAGE_MASK; - npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT; - iommu_release_one(dev, vaddr & PAGE_MASK, npages); -} - -static void iommu_release_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz) -{ - int n; - - while(sz != 0) { - --sz; + struct scatterlist *sg; + int i; - n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT; - iommu_release_one(dev, sg->dma_address & PAGE_MASK, n); + for_each_sg(sgl, sg, nents, i) { + sbus_iommu_unmap_phys(dev, sg->dma_address, sg->length, dir, + attrs); sg->dma_address = 0x21212121; - sg = sg_next(sg); } } #ifdef CONFIG_SBUS -static int iommu_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va, - unsigned long addr, int len) +static void *sbus_iommu_alloc(struct device *dev, size_t len, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { struct iommu_struct *iommu = dev->archdata.iommu; - unsigned long page, end; + unsigned long va, addr, page, end, ret; iopte_t *iopte = iommu->page_table; iopte_t *first; int ioptex; + /* XXX So what is maxphys for us and how do drivers know it? */ + if (!len || len > 256 * 1024) + return NULL; + + len = PAGE_ALIGN(len); + va = __get_free_pages(gfp | __GFP_ZERO, get_order(len)); + if (va == 0) + return NULL; + + addr = ret = sparc_dma_alloc_resource(dev, len); + if (!addr) + goto out_free_pages; + BUG_ON((va & ~PAGE_MASK) != 0); BUG_ON((addr & ~PAGE_MASK) != 0); BUG_ON((len & ~PAGE_MASK) != 0); @@ -341,7 +350,6 @@ static int iommu_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long while(addr < end) { page = va; { - pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; @@ -352,9 +360,8 @@ static int iommu_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long else __flush_page_to_ram(page); - pgdp = pgd_offset(&init_mm, addr); - pmdp = pmd_offset(pgdp, addr); - ptep = pte_offset_map(pmdp, addr); + pmdp = pmd_off_k(addr); + ptep = pte_offset_kernel(pmdp, addr); set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot)); } @@ -379,16 +386,25 @@ static int iommu_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long flush_tlb_all(); iommu_invalidate(iommu->regs); - *pba = iommu->start + (ioptex << PAGE_SHIFT); - return 0; + *dma_handle = iommu->start + (ioptex << PAGE_SHIFT); + return (void *)ret; + +out_free_pages: + free_pages(va, get_order(len)); + return NULL; } -static void iommu_unmap_dma_area(struct device *dev, unsigned long busa, int len) +static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr, + dma_addr_t busa, unsigned long attrs) { struct iommu_struct *iommu = dev->archdata.iommu; iopte_t *iopte = iommu->page_table; - unsigned long end; + struct page *page = virt_to_page(cpu_addr); int ioptex = (busa - iommu->start) >> PAGE_SHIFT; + unsigned long end; + + if (!sparc_dma_free_resource(cpu_addr, len)) + return; BUG_ON((busa & ~PAGE_MASK) != 0); BUG_ON((len & ~PAGE_MASK) != 0); @@ -402,40 +418,35 @@ static void iommu_unmap_dma_area(struct device *dev, unsigned long busa, int len flush_tlb_all(); iommu_invalidate(iommu->regs); bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT); + + __free_pages(page, get_order(len)); } #endif -static const struct sparc32_dma_ops iommu_dma_gflush_ops = { - .get_scsi_one = iommu_get_scsi_one_gflush, - .get_scsi_sgl = iommu_get_scsi_sgl_gflush, - .release_scsi_one = iommu_release_scsi_one, - .release_scsi_sgl = iommu_release_scsi_sgl, +static const struct dma_map_ops sbus_iommu_dma_gflush_ops = { #ifdef CONFIG_SBUS - .map_dma_area = iommu_map_dma_area, - .unmap_dma_area = iommu_unmap_dma_area, + .alloc = sbus_iommu_alloc, + .free = sbus_iommu_free, #endif + .map_phys = sbus_iommu_map_phys_gflush, + .unmap_phys = sbus_iommu_unmap_phys, + .map_sg = sbus_iommu_map_sg_gflush, + .unmap_sg = sbus_iommu_unmap_sg, }; -static const struct sparc32_dma_ops iommu_dma_pflush_ops = { - .get_scsi_one = iommu_get_scsi_one_pflush, - .get_scsi_sgl = iommu_get_scsi_sgl_pflush, - .release_scsi_one = iommu_release_scsi_one, - .release_scsi_sgl = iommu_release_scsi_sgl, +static const struct dma_map_ops sbus_iommu_dma_pflush_ops = { #ifdef CONFIG_SBUS - .map_dma_area = iommu_map_dma_area, - .unmap_dma_area = iommu_unmap_dma_area, + .alloc = sbus_iommu_alloc, + .free = sbus_iommu_free, #endif + .map_phys = sbus_iommu_map_phys_pflush, + .unmap_phys = sbus_iommu_unmap_phys, + .map_sg = sbus_iommu_map_sg_pflush, + .unmap_sg = sbus_iommu_unmap_sg, }; void __init ld_mmu_iommu(void) { - if (flush_page_for_dma_global) { - /* flush_page_for_dma flushes everything, no matter of what page is it */ - sparc32_dma_ops = &iommu_dma_gflush_ops; - } else { - sparc32_dma_ops = &iommu_dma_pflush_ops; - } - if (viking_mxcc_present || srmmu_modtype == HyperSparc) { dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV); ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID; diff --git a/arch/sparc/mm/leon_mm.c b/arch/sparc/mm/leon_mm.c index 5bed085a2c17..1dc9b3d70eda 100644 --- a/arch/sparc/mm/leon_mm.c +++ b/arch/sparc/mm/leon_mm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * linux/arch/sparc/mm/leon_m.c * @@ -15,10 +16,10 @@ #include <asm/leon.h> #include <asm/tlbflush.h> -#include "srmmu.h" +#include "mm_32.h" int leon_flush_during_switch = 1; -int srmmu_swprobe_trace; +static int srmmu_swprobe_trace; static inline unsigned long leon_get_ctable_ptr(void) { @@ -38,12 +39,10 @@ unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr) unsigned int ctxtbl; unsigned int pgd, pmd, ped; unsigned int ptr; - unsigned int lvl, pte, paddrbase; + unsigned int lvl, pte; unsigned int ctx; unsigned int paddr_calc; - paddrbase = 0; - if (srmmu_swprobe_trace) printk(KERN_INFO "swprobe: trace on\n"); @@ -72,7 +71,6 @@ unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr) printk(KERN_INFO "swprobe: pgd is entry level 3\n"); lvl = 3; pte = pgd; - paddrbase = pgd & _SRMMU_PTE_PMASK_LEON; goto ready; } if (((pgd & SRMMU_ET_MASK) != SRMMU_ET_PTD)) { @@ -95,7 +93,6 @@ unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr) printk(KERN_INFO "swprobe: pmd is entry level 2\n"); lvl = 2; pte = pmd; - paddrbase = pmd & _SRMMU_PTE_PMASK_LEON; goto ready; } if (((pmd & SRMMU_ET_MASK) != SRMMU_ET_PTD)) { @@ -123,7 +120,6 @@ unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr) printk(KERN_INFO "swprobe: ped is entry level 1\n"); lvl = 1; pte = ped; - paddrbase = ped & _SRMMU_PTE_PMASK_LEON; goto ready; } if (((ped & SRMMU_ET_MASK) != SRMMU_ET_PTD)) { @@ -146,7 +142,6 @@ unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr) printk(KERN_INFO "swprobe: ptr is entry level 0\n"); lvl = 0; pte = ptr; - paddrbase = ptr & _SRMMU_PTE_PMASK_LEON; goto ready; } if (srmmu_swprobe_trace) diff --git a/arch/sparc/mm/mm_32.h b/arch/sparc/mm/mm_32.h new file mode 100644 index 000000000000..ee55f1080634 --- /dev/null +++ b/arch/sparc/mm/mm_32.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* fault_32.c - visible as they are called from assembler */ +asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, + unsigned long address); + +void window_overflow_fault(void); +void window_underflow_fault(unsigned long sp); +void window_ret_fault(struct pt_regs *regs); + +/* srmmu.c */ +extern char *srmmu_name; +extern int viking_mxcc_present; +extern int flush_page_for_dma_global; + +extern void (*poke_srmmu)(void); + +void __init srmmu_paging_init(void); + +/* iommu.c */ +void ld_mmu_iommu(void); diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index 036c2797dece..f8fb4911d360 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * srmmu.c: SRMMU specific routines for memory management. * @@ -10,10 +11,11 @@ #include <linux/seq_file.h> #include <linux/spinlock.h> -#include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/pagemap.h> #include <linux/vmalloc.h> #include <linux/kdebug.h> +#include <linux/export.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/log2.h> @@ -35,7 +37,6 @@ #include <asm/mbus.h> #include <asm/page.h> #include <asm/asi.h> -#include <asm/msi.h> #include <asm/smp.h> #include <asm/io.h> @@ -48,11 +49,12 @@ #include <asm/mxcc.h> #include <asm/ross.h> -#include "srmmu.h" +#include "mm_32.h" enum mbus_module srmmu_modtype; static unsigned int hwbug_bitmask; int vac_cache_size; +EXPORT_SYMBOL(vac_cache_size); int vac_line_size; extern struct resource sparc_iomap; @@ -62,6 +64,7 @@ extern unsigned long last_valid_pfn; static pgd_t *srmmu_swapper_pg_dir; const struct sparc32_cachetlb_ops *sparc32_cachetlb_ops; +EXPORT_SYMBOL(sparc32_cachetlb_ops); #ifdef CONFIG_SMP const struct sparc32_cachetlb_ops *local_ops; @@ -98,7 +101,6 @@ static unsigned long srmmu_nocache_end; #define SRMMU_NOCACHE_ALIGN_MAX (sizeof(ctxd_t)*SRMMU_MAX_CONTEXTS) void *srmmu_nocache_pool; -void *srmmu_nocache_bitmap; static struct bit_map srmmu_nocache_map; static inline int srmmu_pmd_none(pmd_t pmd) @@ -106,40 +108,36 @@ static inline int srmmu_pmd_none(pmd_t pmd) /* XXX should we hyper_flush_whole_icache here - Anton */ static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp) -{ set_pte((pte_t *)ctxp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pgdp) >> 4))); } - -void pmd_set(pmd_t *pmdp, pte_t *ptep) { - unsigned long ptp; /* Physical address, shifted right by 4 */ - int i; + pte_t pte; - ptp = __nocache_pa((unsigned long) ptep) >> 4; - for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) { - set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); - ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4); - } + pte = __pte((SRMMU_ET_PTD | (__nocache_pa(pgdp) >> 4))); + set_pte((pte_t *)ctxp, pte); } -void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep) -{ - unsigned long ptp; /* Physical address, shifted right by 4 */ - int i; +/* + * Locations of MSI Registers. + */ +#define MSI_MBUS_ARBEN 0xe0001008 /* MBus Arbiter Enable register */ - ptp = page_to_pfn(ptep) << (PAGE_SHIFT-4); /* watch for overflow */ - for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) { - set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); - ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4); - } -} +/* + * Useful bits in the MSI Registers. + */ +#define MSI_ASYNC_MODE 0x80000000 /* Operate the MSI asynchronously */ -/* Find an entry in the third-level page table.. */ -pte_t *pte_offset_kernel(pmd_t *dir, unsigned long address) +static void msi_set_sync(void) { - void *pte; + __asm__ __volatile__ ("lda [%0] %1, %%g3\n\t" + "andn %%g3, %2, %%g3\n\t" + "sta %%g3, [%0] %1\n\t" : : + "r" (MSI_MBUS_ARBEN), + "i" (ASI_M_CTL), "r" (MSI_ASYNC_MODE) : "g3"); +} - pte = __nocache_va((dir->pmdv[0] & SRMMU_PTD_PMASK) << 4); - return (pte_t *) pte + - ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); +void pmd_set(pmd_t *pmdp, pte_t *ptep) +{ + unsigned long ptp = __nocache_pa(ptep) >> 4; + set_pte((pte_t *)&pmd_val(*pmdp), __pte(SRMMU_ET_PTD | ptp)); } /* @@ -149,18 +147,18 @@ pte_t *pte_offset_kernel(pmd_t *dir, unsigned long address) */ static void *__srmmu_get_nocache(int size, int align) { - int offset; + int offset, minsz = 1 << SRMMU_NOCACHE_BITMAP_SHIFT; unsigned long addr; - if (size < SRMMU_NOCACHE_BITMAP_SHIFT) { + if (size < minsz) { printk(KERN_ERR "Size 0x%x too small for nocache request\n", size); - size = SRMMU_NOCACHE_BITMAP_SHIFT; + size = minsz; } - if (size & (SRMMU_NOCACHE_BITMAP_SHIFT - 1)) { - printk(KERN_ERR "Size 0x%x unaligned int nocache request\n", + if (size & (minsz - 1)) { + printk(KERN_ERR "Size 0x%x unaligned in nocache request\n", size); - size += SRMMU_NOCACHE_BITMAP_SHIFT - 1; + size += minsz - 1; } BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX); @@ -171,7 +169,7 @@ static void *__srmmu_get_nocache(int size, int align) printk(KERN_ERR "srmmu: out of nocache %d: %d/%d\n", size, (int) srmmu_nocache_size, srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); - return 0; + return NULL; } addr = SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT); @@ -267,8 +265,11 @@ static void __init srmmu_nocache_calcsize(void) static void __init srmmu_nocache_init(void) { + void *srmmu_nocache_bitmap; unsigned int bitmap_bits; pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; pmd_t *pmd; pte_t *pte; unsigned long paddr, vaddr; @@ -276,13 +277,13 @@ static void __init srmmu_nocache_init(void) bitmap_bits = srmmu_nocache_size >> SRMMU_NOCACHE_BITMAP_SHIFT; - srmmu_nocache_pool = __alloc_bootmem(srmmu_nocache_size, - SRMMU_NOCACHE_ALIGN_MAX, 0UL); + srmmu_nocache_pool = memblock_alloc_or_panic(srmmu_nocache_size, + SRMMU_NOCACHE_ALIGN_MAX); memset(srmmu_nocache_pool, 0, srmmu_nocache_size); srmmu_nocache_bitmap = - __alloc_bootmem(BITS_TO_LONGS(bitmap_bits) * sizeof(long), - SMP_CACHE_BYTES, 0UL); + memblock_alloc_or_panic(BITS_TO_LONGS(bitmap_bits) * sizeof(long), + SMP_CACHE_BYTES); bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits); srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); @@ -296,7 +297,9 @@ static void __init srmmu_nocache_init(void) while (vaddr < srmmu_nocache_end) { pgd = pgd_offset_k(vaddr); - pmd = pmd_offset(__nocache_fix(pgd), vaddr); + p4d = p4d_offset(pgd, vaddr); + pud = pud_offset(p4d, vaddr); + pmd = pmd_offset(__nocache_fix(pud), vaddr); pte = pte_offset_kernel(__nocache_fix(pmd), vaddr); pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV); @@ -337,30 +340,36 @@ pgd_t *get_pgd_fast(void) * Alignments up to the page size are the same for physical and virtual * addresses of the nocache area. */ -pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) +pgtable_t pte_alloc_one(struct mm_struct *mm) { - unsigned long pte; + pte_t *ptep; struct page *page; - if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0) + if (!(ptep = pte_alloc_one_kernel(mm))) return NULL; - page = pfn_to_page(__nocache_pa(pte) >> PAGE_SHIFT); - pgtable_page_ctor(page); - return page; + page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT); + spin_lock(&mm->page_table_lock); + if (page_ref_inc_return(page) == 2 && + !pagetable_pte_ctor(mm, page_ptdesc(page))) { + page_ref_dec(page); + ptep = NULL; + } + spin_unlock(&mm->page_table_lock); + + return ptep; } -void pte_free(struct mm_struct *mm, pgtable_t pte) +void pte_free(struct mm_struct *mm, pgtable_t ptep) { - unsigned long p; + struct page *page; - pgtable_page_dtor(pte); - p = (unsigned long)page_address(pte); /* Cached address (for test) */ - if (p == 0) - BUG(); - p = page_to_pfn(pte) << PAGE_SHIFT; /* Physical address */ + page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT); + spin_lock(&mm->page_table_lock); + if (page_ref_dec_return(page) == 1) + pagetable_dtor(page_ptdesc(page)); + spin_unlock(&mm->page_table_lock); - /* free non cached virtual address*/ - srmmu_free_nocache(__nocache_va(p), PTE_SIZE); + srmmu_free_nocache(ptep, SRMMU_PTE_TABLE_SIZE); } /* context handling - a dynamically sized pool is used */ @@ -437,7 +446,7 @@ static void __init sparc_context_init(int numctx) unsigned long size; size = numctx * sizeof(struct ctx_list); - ctx_list_pool = __alloc_bootmem(size, SMP_CACHE_BYTES, 0UL); + ctx_list_pool = memblock_alloc_or_panic(size, SMP_CACHE_BYTES); for (ctx = 0; ctx < numctx; ctx++) { struct ctx_list *clist; @@ -455,10 +464,12 @@ static void __init sparc_context_init(int numctx) void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, struct task_struct *tsk) { + unsigned long flags; + if (mm->context == NO_CONTEXT) { - spin_lock(&srmmu_context_spinlock); + spin_lock_irqsave(&srmmu_context_spinlock, flags); alloc_context(old_mm, mm); - spin_unlock(&srmmu_context_spinlock); + spin_unlock_irqrestore(&srmmu_context_spinlock, flags); srmmu_ctxd_set(&srmmu_context_table[mm->context], mm->pgd); } @@ -476,13 +487,17 @@ static inline void srmmu_mapioaddr(unsigned long physaddr, unsigned long virt_addr, int bus_type) { pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; unsigned long tmp; physaddr &= PAGE_MASK; pgdp = pgd_offset_k(virt_addr); - pmdp = pmd_offset(pgdp, virt_addr); + p4dp = p4d_offset(pgdp, virt_addr); + pudp = pud_offset(p4dp, virt_addr); + pmdp = pmd_offset(pudp, virt_addr); ptep = pte_offset_kernel(pmdp, virt_addr); tmp = (physaddr >> 4) | SRMMU_ET_PTE; @@ -511,11 +526,16 @@ void srmmu_mapiorange(unsigned int bus, unsigned long xpa, static inline void srmmu_unmapioaddr(unsigned long virt_addr) { pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; + pgdp = pgd_offset_k(virt_addr); - pmdp = pmd_offset(pgdp, virt_addr); + p4dp = p4d_offset(pgdp, virt_addr); + pudp = pud_offset(p4dp, virt_addr); + pmdp = pmd_offset(pudp, virt_addr); ptep = pte_offset_kernel(pmdp, virt_addr); /* No need to flush uncacheable page. */ @@ -653,21 +673,25 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start, unsigned long end) { pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; while (start < end) { pgdp = pgd_offset_k(start); - if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { + p4dp = p4d_offset(pgdp, start); + pudp = pud_offset(p4dp, start); + if (pud_none(*__nocache_fix(pudp))) { pmdp = __srmmu_get_nocache( SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); - pgd_set(__nocache_fix(pgdp), pmdp); + pud_set(__nocache_fix(pudp), pmdp); } - pmdp = pmd_offset(__nocache_fix(pgdp), start); - if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { + pmdp = pmd_offset(__nocache_fix(pudp), start); + if (srmmu_pmd_none(*__nocache_fix(pmdp))) { ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); @@ -684,19 +708,23 @@ static void __init srmmu_allocate_ptable_skeleton(unsigned long start, unsigned long end) { pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; while (start < end) { pgdp = pgd_offset_k(start); - if (pgd_none(*pgdp)) { + p4dp = p4d_offset(pgdp, start); + pudp = pud_offset(p4dp, start); + if (pud_none(*pudp)) { pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE); - pgd_set(pgdp, pmdp); + pud_set((pud_t *)pgdp, pmdp); } - pmdp = pmd_offset(pgdp, start); + pmdp = pmd_offset(pudp, start); if (srmmu_pmd_none(*pmdp)) { ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); @@ -723,7 +751,7 @@ static inline unsigned long srmmu_probe(unsigned long vaddr) "=r" (retval) : "r" (vaddr | 0x400), "i" (ASI_M_FLUSH_PROBE)); } else { - retval = leon_swprobe(vaddr, 0); + retval = leon_swprobe(vaddr, NULL); } return retval; } @@ -739,6 +767,8 @@ static void __init srmmu_inherit_prom_mappings(unsigned long start, unsigned long probed; unsigned long addr; pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; pmd_t *pmdp; pte_t *ptep; int what; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */ @@ -759,53 +789,47 @@ static void __init srmmu_inherit_prom_mappings(unsigned long start, what = 0; addr = start - PAGE_SIZE; - if (!(start & ~(SRMMU_REAL_PMD_MASK))) { - if (srmmu_probe(addr + SRMMU_REAL_PMD_SIZE) == probed) + if (!(start & ~(PMD_MASK))) { + if (srmmu_probe(addr + PMD_SIZE) == probed) what = 1; } - if (!(start & ~(SRMMU_PGDIR_MASK))) { - if (srmmu_probe(addr + SRMMU_PGDIR_SIZE) == probed) + if (!(start & ~(PGDIR_MASK))) { + if (srmmu_probe(addr + PGDIR_SIZE) == probed) what = 2; } pgdp = pgd_offset_k(start); + p4dp = p4d_offset(pgdp, start); + pudp = pud_offset(p4dp, start); if (what == 2) { - *(pgd_t *)__nocache_fix(pgdp) = __pgd(probed); - start += SRMMU_PGDIR_SIZE; + *__nocache_fix(pgdp) = __pgd(probed); + start += PGDIR_SIZE; continue; } - if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { + if (pud_none(*__nocache_fix(pudp))) { pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); - pgd_set(__nocache_fix(pgdp), pmdp); + pud_set(__nocache_fix(pudp), pmdp); } - pmdp = pmd_offset(__nocache_fix(pgdp), start); - if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { + pmdp = pmd_offset(__nocache_fix(pudp), start); + if (what == 1) { + *(pmd_t *)__nocache_fix(pmdp) = __pmd(probed); + start += PMD_SIZE; + continue; + } + if (srmmu_pmd_none(*__nocache_fix(pmdp))) { ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); memset(__nocache_fix(ptep), 0, PTE_SIZE); pmd_set(__nocache_fix(pmdp), ptep); } - if (what == 1) { - /* We bend the rule where all 16 PTPs in a pmd_t point - * inside the same PTE page, and we leak a perfectly - * good hardware PTE piece. Alternatives seem worse. - */ - unsigned int x; /* Index of HW PMD in soft cluster */ - unsigned long *val; - x = (start >> PMD_SHIFT) & 15; - val = &pmdp->pmdv[x]; - *(unsigned long *)__nocache_fix(val) = probed; - start += SRMMU_REAL_PMD_SIZE; - continue; - } ptep = pte_offset_kernel(__nocache_fix(pmdp), start); - *(pte_t *)__nocache_fix(ptep) = __pte(probed); + *__nocache_fix(ptep) = __pte(probed); start += PAGE_SIZE; } } @@ -819,15 +843,15 @@ static void __init do_large_mapping(unsigned long vaddr, unsigned long phys_base unsigned long big_pte; big_pte = KERNEL_PTE(phys_base >> 4); - *(pgd_t *)__nocache_fix(pgdp) = __pgd(big_pte); + *__nocache_fix(pgdp) = __pgd(big_pte); } /* Map sp_bank entry SP_ENTRY, starting at virtual address VBASE. */ static unsigned long __init map_spbank(unsigned long vbase, int sp_entry) { - unsigned long pstart = (sp_banks[sp_entry].base_addr & SRMMU_PGDIR_MASK); - unsigned long vstart = (vbase & SRMMU_PGDIR_MASK); - unsigned long vend = SRMMU_PGDIR_ALIGN(vbase + sp_banks[sp_entry].num_bytes); + unsigned long pstart = (sp_banks[sp_entry].base_addr & PGDIR_MASK); + unsigned long vstart = (vbase & PGDIR_MASK); + unsigned long vend = PGDIR_ALIGN(vbase + sp_banks[sp_entry].num_bytes); /* Map "low" memory only */ const unsigned long min_vaddr = PAGE_OFFSET; const unsigned long max_vaddr = PAGE_OFFSET + SRMMU_MAXMEM; @@ -840,7 +864,7 @@ static unsigned long __init map_spbank(unsigned long vbase, int sp_entry) while (vstart < vend) { do_large_mapping(vstart, pstart); - vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE; + vstart += PGDIR_SIZE; pstart += PGDIR_SIZE; } return vstart; } @@ -858,9 +882,7 @@ static void __init map_kernel(void) } } -void (*poke_srmmu)(void) __cpuinitdata = NULL; - -extern unsigned long bootmem_init(unsigned long *pages_avail); +void (*poke_srmmu)(void) = NULL; void __init srmmu_paging_init(void) { @@ -868,6 +890,8 @@ void __init srmmu_paging_init(void) phandle cpunode; char node_str[128]; pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; pmd_t *pmd; pte_t *pte; unsigned long pages_avail; @@ -906,10 +930,10 @@ void __init srmmu_paging_init(void) /* ctx table has to be physically aligned to its size */ srmmu_context_table = __srmmu_get_nocache(num_contexts * sizeof(ctxd_t), num_contexts * sizeof(ctxd_t)); - srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa((unsigned long)srmmu_context_table); + srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa(srmmu_context_table); for (i = 0; i < num_contexts; i++) - srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir); + srmmu_ctxd_set(__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir); flush_cache_all(); srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys); @@ -929,7 +953,9 @@ void __init srmmu_paging_init(void) srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_END); pgd = pgd_offset_k(PKMAP_BASE); - pmd = pmd_offset(pgd, PKMAP_BASE); + p4d = p4d_offset(pgd, PKMAP_BASE); + pud = pud_offset(p4d, PKMAP_BASE); + pmd = pmd_offset(pud, PKMAP_BASE); pte = pte_offset_kernel(pmd, PKMAP_BASE); pkmap_page_table = pte; @@ -938,27 +964,14 @@ void __init srmmu_paging_init(void) sparc_context_init(num_contexts); - kmap_init(); - { - unsigned long zones_size[MAX_NR_ZONES]; - unsigned long zholes_size[MAX_NR_ZONES]; - unsigned long npages; - int znum; - - for (znum = 0; znum < MAX_NR_ZONES; znum++) - zones_size[znum] = zholes_size[znum] = 0; - - npages = max_low_pfn - pfn_base; + unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 }; - zones_size[ZONE_DMA] = npages; - zholes_size[ZONE_DMA] = npages - pages_avail; + max_zone_pfn[ZONE_DMA] = max_low_pfn; + max_zone_pfn[ZONE_NORMAL] = max_low_pfn; + max_zone_pfn[ZONE_HIGHMEM] = highend_pfn; - npages = highend_pfn - max_low_pfn; - zones_size[ZONE_HIGHMEM] = npages; - zholes_size[ZONE_HIGHMEM] = npages - calc_highpages(); - - free_area_init_node(0, zones_size, pfn_base, zholes_size); + free_area_init(max_zone_pfn); } } @@ -983,14 +996,15 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) void destroy_context(struct mm_struct *mm) { + unsigned long flags; if (mm->context != NO_CONTEXT) { flush_cache_mm(mm); srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir); flush_tlb_mm(mm); - spin_lock(&srmmu_context_spinlock); + spin_lock_irqsave(&srmmu_context_spinlock, flags); free_context(mm->context); - spin_unlock(&srmmu_context_spinlock); + spin_unlock_irqrestore(&srmmu_context_spinlock, flags); mm->context = NO_CONTEXT; } } @@ -1055,7 +1069,7 @@ static void __init init_vac_layout(void) (int)vac_cache_size, (int)vac_line_size); } -static void __cpuinit poke_hypersparc(void) +static void poke_hypersparc(void) { volatile unsigned long clear; unsigned long mreg = srmmu_get_mmureg(); @@ -1107,7 +1121,7 @@ static void __init init_hypersparc(void) hypersparc_setup_blockops(); } -static void __cpuinit poke_swift(void) +static void poke_swift(void) { unsigned long mreg; @@ -1287,7 +1301,7 @@ static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long } -static void __cpuinit poke_turbosparc(void) +static void poke_turbosparc(void) { unsigned long mreg = srmmu_get_mmureg(); unsigned long ccreg; @@ -1350,7 +1364,7 @@ static void __init init_turbosparc(void) poke_srmmu = poke_turbosparc; } -static void __cpuinit poke_tsunami(void) +static void poke_tsunami(void) { unsigned long mreg = srmmu_get_mmureg(); @@ -1391,7 +1405,7 @@ static void __init init_tsunami(void) tsunami_setup_blockops(); } -static void __cpuinit poke_viking(void) +static void poke_viking(void) { unsigned long mreg = srmmu_get_mmureg(); static int smp_catch; @@ -1433,7 +1447,7 @@ static void __cpuinit poke_viking(void) srmmu_set_mmureg(mreg); } -static struct sparc32_cachetlb_ops viking_ops = { +static struct sparc32_cachetlb_ops viking_ops __ro_after_init = { .cache_all = viking_flush_cache_all, .cache_mm = viking_flush_cache_mm, .cache_page = viking_flush_cache_page, @@ -1464,7 +1478,7 @@ static struct sparc32_cachetlb_ops viking_ops = { * flushes going at once will require SMP locking anyways so there's * no real value in trying any harder than this. */ -static struct sparc32_cachetlb_ops viking_sun4d_smp_ops = { +static struct sparc32_cachetlb_ops viking_sun4d_smp_ops __ro_after_init = { .cache_all = viking_flush_cache_all, .cache_mm = viking_flush_cache_mm, .cache_page = viking_flush_cache_page, @@ -1491,7 +1505,7 @@ static void __init init_viking(void) /* * We need this to make sure old viking takes no hits - * on it's cache for dma snoops to workaround the + * on its cache for dma snoops to workaround the * "load from non-cacheable memory" interrupt bug. * This is only necessary because of the new way in * which we use the IOMMU. @@ -1615,30 +1629,32 @@ static void __init get_srmmu_type(void) /* Local cross-calls. */ static void smp_flush_page_for_dma(unsigned long page) { - xc1((smpfunc_t) local_ops->page_for_dma, page); + xc1(local_ops->page_for_dma, page); local_ops->page_for_dma(page); } static void smp_flush_cache_all(void) { - xc0((smpfunc_t) local_ops->cache_all); + xc0(local_ops->cache_all); local_ops->cache_all(); } static void smp_flush_tlb_all(void) { - xc0((smpfunc_t) local_ops->tlb_all); + xc0(local_ops->tlb_all); local_ops->tlb_all(); } +static bool any_other_mm_cpus(struct mm_struct *mm) +{ + return cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids; +} + static void smp_flush_cache_mm(struct mm_struct *mm) { if (mm->context != NO_CONTEXT) { - cpumask_t cpu_mask; - cpumask_copy(&cpu_mask, mm_cpumask(mm)); - cpumask_clear_cpu(smp_processor_id(), &cpu_mask); - if (!cpumask_empty(&cpu_mask)) - xc1((smpfunc_t) local_ops->cache_mm, (unsigned long) mm); + if (any_other_mm_cpus(mm)) + xc1(local_ops->cache_mm, (unsigned long)mm); local_ops->cache_mm(mm); } } @@ -1646,11 +1662,8 @@ static void smp_flush_cache_mm(struct mm_struct *mm) static void smp_flush_tlb_mm(struct mm_struct *mm) { if (mm->context != NO_CONTEXT) { - cpumask_t cpu_mask; - cpumask_copy(&cpu_mask, mm_cpumask(mm)); - cpumask_clear_cpu(smp_processor_id(), &cpu_mask); - if (!cpumask_empty(&cpu_mask)) { - xc1((smpfunc_t) local_ops->tlb_mm, (unsigned long) mm); + if (any_other_mm_cpus(mm)) { + xc1(local_ops->tlb_mm, (unsigned long)mm); if (atomic_read(&mm->mm_users) == 1 && current->active_mm == mm) cpumask_copy(mm_cpumask(mm), cpumask_of(smp_processor_id())); @@ -1666,12 +1679,9 @@ static void smp_flush_cache_range(struct vm_area_struct *vma, struct mm_struct *mm = vma->vm_mm; if (mm->context != NO_CONTEXT) { - cpumask_t cpu_mask; - cpumask_copy(&cpu_mask, mm_cpumask(mm)); - cpumask_clear_cpu(smp_processor_id(), &cpu_mask); - if (!cpumask_empty(&cpu_mask)) - xc3((smpfunc_t) local_ops->cache_range, - (unsigned long) vma, start, end); + if (any_other_mm_cpus(mm)) + xc3(local_ops->cache_range, (unsigned long)vma, start, + end); local_ops->cache_range(vma, start, end); } } @@ -1683,12 +1693,9 @@ static void smp_flush_tlb_range(struct vm_area_struct *vma, struct mm_struct *mm = vma->vm_mm; if (mm->context != NO_CONTEXT) { - cpumask_t cpu_mask; - cpumask_copy(&cpu_mask, mm_cpumask(mm)); - cpumask_clear_cpu(smp_processor_id(), &cpu_mask); - if (!cpumask_empty(&cpu_mask)) - xc3((smpfunc_t) local_ops->tlb_range, - (unsigned long) vma, start, end); + if (any_other_mm_cpus(mm)) + xc3(local_ops->tlb_range, (unsigned long)vma, start, + end); local_ops->tlb_range(vma, start, end); } } @@ -1698,12 +1705,8 @@ static void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page) struct mm_struct *mm = vma->vm_mm; if (mm->context != NO_CONTEXT) { - cpumask_t cpu_mask; - cpumask_copy(&cpu_mask, mm_cpumask(mm)); - cpumask_clear_cpu(smp_processor_id(), &cpu_mask); - if (!cpumask_empty(&cpu_mask)) - xc2((smpfunc_t) local_ops->cache_page, - (unsigned long) vma, page); + if (any_other_mm_cpus(mm)) + xc2(local_ops->cache_page, (unsigned long)vma, page); local_ops->cache_page(vma, page); } } @@ -1713,12 +1716,8 @@ static void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) struct mm_struct *mm = vma->vm_mm; if (mm->context != NO_CONTEXT) { - cpumask_t cpu_mask; - cpumask_copy(&cpu_mask, mm_cpumask(mm)); - cpumask_clear_cpu(smp_processor_id(), &cpu_mask); - if (!cpumask_empty(&cpu_mask)) - xc2((smpfunc_t) local_ops->tlb_page, - (unsigned long) vma, page); + if (any_other_mm_cpus(mm)) + xc2(local_ops->tlb_page, (unsigned long)vma, page); local_ops->tlb_page(vma, page); } } @@ -1732,23 +1731,19 @@ static void smp_flush_page_to_ram(unsigned long page) * XXX This experiment failed, research further... -DaveM */ #if 1 - xc1((smpfunc_t) local_ops->page_to_ram, page); + xc1(local_ops->page_to_ram, page); #endif local_ops->page_to_ram(page); } static void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) { - cpumask_t cpu_mask; - cpumask_copy(&cpu_mask, mm_cpumask(mm)); - cpumask_clear_cpu(smp_processor_id(), &cpu_mask); - if (!cpumask_empty(&cpu_mask)) - xc2((smpfunc_t) local_ops->sig_insns, - (unsigned long) mm, insn_addr); + if (any_other_mm_cpus(mm)) + xc2(local_ops->sig_insns, (unsigned long)mm, insn_addr); local_ops->sig_insns(mm, insn_addr); } -static struct sparc32_cachetlb_ops smp_cachetlb_ops = { +static struct sparc32_cachetlb_ops smp_cachetlb_ops __ro_after_init = { .cache_all = smp_flush_cache_all, .cache_mm = smp_flush_cache_mm, .cache_page = smp_flush_cache_page, @@ -1766,9 +1761,6 @@ static struct sparc32_cachetlb_ops smp_cachetlb_ops = { /* Load up routines and constants for sun4m and sun4d mmu */ void __init load_mmu(void) { - extern void ld_mmu_iommu(void); - extern void ld_mmu_iounit(void); - /* Functions */ get_srmmu_type(); @@ -1800,9 +1792,7 @@ void __init load_mmu(void) &smp_cachetlb_ops; #endif - if (sparc_cpu_model == sun4d) - ld_mmu_iounit(); - else + if (sparc_cpu_model != sun4d) ld_mmu_iommu(); #ifdef CONFIG_SMP if (sparc_cpu_model == sun4d) diff --git a/arch/sparc/mm/srmmu.h b/arch/sparc/mm/srmmu.h deleted file mode 100644 index 5703274ccf89..000000000000 --- a/arch/sparc/mm/srmmu.h +++ /dev/null @@ -1,4 +0,0 @@ -/* srmmu.c */ -extern char *srmmu_name; - -extern void (*poke_srmmu)(void); diff --git a/arch/sparc/mm/srmmu_access.S b/arch/sparc/mm/srmmu_access.S index d0a67b2c2383..d8d2e644a5ca 100644 --- a/arch/sparc/mm/srmmu_access.S +++ b/arch/sparc/mm/srmmu_access.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Assembler variants of srmmu access functions. * Implemented in assembler to allow run-time patching. * LEON uses a different ASI for MMUREGS than SUN. diff --git a/arch/sparc/mm/swift.S b/arch/sparc/mm/swift.S index 5d2b88d39424..f414bfd8d899 100644 --- a/arch/sparc/mm/swift.S +++ b/arch/sparc/mm/swift.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * swift.S: MicroSparc-II mmu/cache operations. * diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c index 7a91f288c708..a35ddcca5e76 100644 --- a/arch/sparc/mm/tlb.c +++ b/arch/sparc/mm/tlb.c @@ -1,17 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 /* arch/sparc64/mm/tlb.c * * Copyright (C) 2004 David S. Miller <davem@redhat.com> */ #include <linux/kernel.h> -#include <linux/init.h> #include <linux/percpu.h> #include <linux/mm.h> #include <linux/swap.h> #include <linux/preempt.h> +#include <linux/pagemap.h> -#include <asm/pgtable.h> -#include <asm/pgalloc.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> #include <asm/mmu_context.h> @@ -53,22 +52,25 @@ out: void arch_enter_lazy_mmu_mode(void) { - struct tlb_batch *tb = &__get_cpu_var(tlb_batch); + struct tlb_batch *tb; + preempt_disable(); + tb = this_cpu_ptr(&tlb_batch); tb->active = 1; } void arch_leave_lazy_mmu_mode(void) { - struct tlb_batch *tb = &__get_cpu_var(tlb_batch); + struct tlb_batch *tb = this_cpu_ptr(&tlb_batch); if (tb->tlb_nr) flush_tlb_pending(); tb->active = 0; + preempt_enable(); } static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, - bool exec) + bool exec, unsigned int hugepage_shift) { struct tlb_batch *tb = &get_cpu_var(tlb_batch); unsigned long nr; @@ -85,13 +87,21 @@ static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, } if (!tb->active) { - flush_tsb_user_page(mm, vaddr); + flush_tsb_user_page(mm, vaddr, hugepage_shift); global_flush_tlb_page(mm, vaddr); goto out; } - if (nr == 0) + if (nr == 0) { tb->mm = mm; + tb->hugepage_shift = hugepage_shift; + } + + if (tb->hugepage_shift != hugepage_shift) { + flush_tlb_pending(); + tb->hugepage_shift = hugepage_shift; + nr = 0; + } tb->vaddrs[nr] = vaddr; tb->tlb_nr = ++nr; @@ -103,13 +113,15 @@ out: } void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, - pte_t *ptep, pte_t orig, int fullmm) + pte_t *ptep, pte_t orig, int fullmm, + unsigned int hugepage_shift) { if (tlb_type != hypervisor && pte_dirty(orig)) { unsigned long paddr, pfn = pte_pfn(orig); struct address_space *mapping; struct page *page; + struct folio *folio; if (!pfn_valid(pfn)) goto no_cache_flush; @@ -119,53 +131,71 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, goto no_cache_flush; /* A real file page? */ - mapping = page_mapping(page); + folio = page_folio(page); + mapping = folio_flush_mapping(folio); if (!mapping) goto no_cache_flush; paddr = (unsigned long) page_address(page); if ((paddr ^ vaddr) & (1 << 13)) - flush_dcache_page_all(mm, page); + flush_dcache_folio_all(mm, folio); } no_cache_flush: if (!fullmm) - tlb_batch_add_one(mm, vaddr, pte_exec(orig)); + tlb_batch_add_one(mm, vaddr, pte_exec(orig), hugepage_shift); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr, - pmd_t pmd, bool exec) + pmd_t pmd) { unsigned long end; pte_t *pte; pte = pte_offset_map(&pmd, vaddr); + if (!pte) + return; end = vaddr + HPAGE_SIZE; while (vaddr < end) { - if (pte_val(*pte) & _PAGE_VALID) - tlb_batch_add_one(mm, vaddr, exec); + if (pte_val(*pte) & _PAGE_VALID) { + bool exec = pte_exec(*pte); + + tlb_batch_add_one(mm, vaddr, exec, PAGE_SHIFT); + } pte++; vaddr += PAGE_SIZE; } pte_unmap(pte); } -void set_pmd_at(struct mm_struct *mm, unsigned long addr, - pmd_t *pmdp, pmd_t pmd) -{ - pmd_t orig = *pmdp; - - *pmdp = pmd; +static void __set_pmd_acct(struct mm_struct *mm, unsigned long addr, + pmd_t orig, pmd_t pmd) +{ if (mm == &init_mm) return; - if ((pmd_val(pmd) ^ pmd_val(orig)) & PMD_ISHUGE) { - if (pmd_val(pmd) & PMD_ISHUGE) - mm->context.huge_pte_count++; - else - mm->context.huge_pte_count--; + if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) { + /* + * Note that this routine only sets pmds for THP pages. + * Hugetlb pages are handled elsewhere. We need to check + * for huge zero page. Huge zero pages are like hugetlb + * pages in that there is no RSS, but there is the need + * for TSB entries. So, huge zero page counts go into + * hugetlb_pte_count. + */ + if (pmd_val(pmd) & _PAGE_PMD_HUGE) { + if (is_huge_zero_pmd(pmd)) + mm->context.hugetlb_pte_count++; + else + mm->context.thp_pte_count++; + } else { + if (is_huge_zero_pmd(orig)) + mm->context.hugetlb_pte_count--; + else + mm->context.thp_pte_count--; + } /* Do not try to allocate the TSB hash table if we * don't have one already. We have various locks held @@ -178,16 +208,67 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, } if (!pmd_none(orig)) { - bool exec = ((pmd_val(orig) & PMD_HUGE_EXEC) != 0); - addr &= HPAGE_MASK; - if (pmd_val(orig) & PMD_ISHUGE) - tlb_batch_add_one(mm, addr, exec); - else - tlb_batch_pmd_scan(mm, addr, orig, exec); + if (pmd_trans_huge(orig)) { + pte_t orig_pte = __pte(pmd_val(orig)); + bool exec = pte_exec(orig_pte); + + tlb_batch_add_one(mm, addr, exec, REAL_HPAGE_SHIFT); + tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec, + REAL_HPAGE_SHIFT); + } else { + tlb_batch_pmd_scan(mm, addr, orig); + } } } +void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + pmd_t orig = *pmdp; + + *pmdp = pmd; + __set_pmd_acct(mm, addr, orig, pmd); +} + +static inline pmd_t pmdp_establish(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, pmd_t pmd) +{ + pmd_t old; + + do { + old = *pmdp; + } while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd); + __set_pmd_acct(vma->vm_mm, address, old, pmd); + + return old; +} + +/* + * This routine is only called when splitting a THP + */ +pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmdp) +{ + pmd_t old, entry; + + VM_WARN_ON_ONCE(!pmd_present(*pmdp)); + entry = __pmd(pmd_val(*pmdp) & ~_PAGE_VALID); + old = pmdp_establish(vma, address, pmdp, entry); + flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); + + /* + * set_pmd_at() will not be called in a way to decrement + * thp_pte_count when splitting a THP, so do it now. + * Sanity check pmd before doing the actual decrement. + */ + if ((pmd_val(entry) & _PAGE_PMD_HUGE) && + !is_huge_zero_pmd(entry)) + (vma->vm_mm)->context.thp_pte_count--; + + return old; +} + void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) { @@ -196,11 +277,11 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, assert_spin_locked(&mm->page_table_lock); /* FIFO */ - if (!mm->pmd_huge_pte) + if (!pmd_huge_pte(mm, pmdp)) INIT_LIST_HEAD(lh); else - list_add(lh, (struct list_head *) mm->pmd_huge_pte); - mm->pmd_huge_pte = pgtable; + list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); + pmd_huge_pte(mm, pmdp) = pgtable; } pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) @@ -211,12 +292,12 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) assert_spin_locked(&mm->page_table_lock); /* FIFO */ - pgtable = mm->pmd_huge_pte; + pgtable = pmd_huge_pte(mm, pmdp); lh = (struct list_head *) pgtable; if (list_empty(lh)) - mm->pmd_huge_pte = NULL; + pmd_huge_pte(mm, pmdp) = NULL; else { - mm->pmd_huge_pte = (pgtable_t) lh->next; + pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; list_del(lh); } pte_val(pgtable[0]) = 0; diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c index 2cc3bce5ee91..5fe52a64c7e7 100644 --- a/arch/sparc/mm/tsb.c +++ b/arch/sparc/mm/tsb.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* arch/sparc64/mm/tsb.c * * Copyright (C) 2006, 2008 David S. Miller <davem@davemloft.net> @@ -6,9 +7,12 @@ #include <linux/kernel.h> #include <linux/preempt.h> #include <linux/slab.h> +#include <linux/mm_types.h> +#include <linux/pgtable.h> + #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/mmu_context.h> +#include <asm/setup.h> #include <asm/tsb.h> #include <asm/tlb.h> #include <asm/oplib.h> @@ -26,6 +30,20 @@ static inline int tag_compare(unsigned long tag, unsigned long vaddr) return (tag == (vaddr >> 22)); } +static void flush_tsb_kernel_range_scan(unsigned long start, unsigned long end) +{ + unsigned long idx; + + for (idx = 0; idx < KERNEL_TSB_NENTRIES; idx++) { + struct tsb *ent = &swapper_tsb[idx]; + unsigned long match = idx << 13; + + match |= (ent->tag << 22); + if (match >= start && match < end) + ent->tag = (1UL << TSB_TAG_INVALID_BIT); + } +} + /* TSB flushes need only occur on the processor initiating the address * space modification, not on each cpu the address space has run on. * Only the TLB flush needs that treatment. @@ -35,6 +53,9 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end) { unsigned long v; + if ((end - start) >> PAGE_SHIFT >= 2 * KERNEL_TSB_NENTRIES) + return flush_tsb_kernel_range_scan(start, end); + for (v = start; v < end; v += PAGE_SIZE) { unsigned long hash = tsb_hash(v, PAGE_SHIFT, KERNEL_TSB_NENTRIES); @@ -68,6 +89,33 @@ static void __flush_tsb_one(struct tlb_batch *tb, unsigned long hash_shift, __flush_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, nentries); } +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +static void __flush_huge_tsb_one_entry(unsigned long tsb, unsigned long v, + unsigned long hash_shift, + unsigned long nentries, + unsigned int hugepage_shift) +{ + unsigned int hpage_entries; + unsigned int i; + + hpage_entries = 1 << (hugepage_shift - hash_shift); + for (i = 0; i < hpage_entries; i++) + __flush_tsb_one_entry(tsb, v + (i << hash_shift), hash_shift, + nentries); +} + +static void __flush_huge_tsb_one(struct tlb_batch *tb, unsigned long hash_shift, + unsigned long tsb, unsigned long nentries, + unsigned int hugepage_shift) +{ + unsigned long i; + + for (i = 0; i < tb->tlb_nr; i++) + __flush_huge_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, + nentries, hugepage_shift); +} +#endif + void flush_tsb_user(struct tlb_batch *tb) { struct mm_struct *mm = tb->mm; @@ -75,43 +123,61 @@ void flush_tsb_user(struct tlb_batch *tb) spin_lock_irqsave(&mm->context.lock, flags); - base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; - nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; - if (tlb_type == cheetah_plus || tlb_type == hypervisor) - base = __pa(base); - __flush_tsb_one(tb, PAGE_SHIFT, base, nentries); - + if (tb->hugepage_shift < REAL_HPAGE_SHIFT) { + base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; + nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + if (tb->hugepage_shift == PAGE_SHIFT) + __flush_tsb_one(tb, PAGE_SHIFT, base, nentries); +#if defined(CONFIG_HUGETLB_PAGE) + else + __flush_huge_tsb_one(tb, PAGE_SHIFT, base, nentries, + tb->hugepage_shift); +#endif + } #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { + else if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) base = __pa(base); - __flush_tsb_one(tb, HPAGE_SHIFT, base, nentries); + __flush_huge_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries, + tb->hugepage_shift); } #endif spin_unlock_irqrestore(&mm->context.lock, flags); } -void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr) +void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, + unsigned int hugepage_shift) { unsigned long nentries, base, flags; spin_lock_irqsave(&mm->context.lock, flags); - base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; - nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; - if (tlb_type == cheetah_plus || tlb_type == hypervisor) - base = __pa(base); - __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries); - + if (hugepage_shift < REAL_HPAGE_SHIFT) { + base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; + nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + if (hugepage_shift == PAGE_SHIFT) + __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, + nentries); +#if defined(CONFIG_HUGETLB_PAGE) + else + __flush_huge_tsb_one_entry(base, vaddr, PAGE_SHIFT, + nentries, hugepage_shift); +#endif + } #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { + else if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) base = __pa(base); - __flush_tsb_one_entry(base, vaddr, HPAGE_SHIFT, nentries); + __flush_huge_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, + nentries, hugepage_shift); } #endif spin_unlock_irqrestore(&mm->context.lock, flags); @@ -133,7 +199,19 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign mm->context.tsb_block[tsb_idx].tsb_nentries = tsb_bytes / sizeof(struct tsb); - base = TSBMAP_BASE; + switch (tsb_idx) { + case MM_TSB_BASE: + base = TSBMAP_8K_BASE; + break; +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + case MM_TSB_HUGE: + base = TSBMAP_4M_BASE; + break; +#endif + default: + BUG(); + } + tte = pgprot_val(PAGE_KERNEL_LOCKED); tsb_paddr = __pa(mm->context.tsb_block[tsb_idx].tsb); BUG_ON(tsb_paddr & (tsb_bytes - 1UL)); @@ -188,7 +266,7 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign default: printk(KERN_ERR "TSB[%s:%d]: Impossible TSB size %lu, killing process.\n", current->comm, current->pid, tsb_bytes); - do_exit(SIGSEGV); + BUG(); } tte |= pte_sz_bits(page_sz); @@ -273,7 +351,7 @@ void __init pgtable_cache_init(void) prom_halt(); } - for (i = 0; i < 8; i++) { + for (i = 0; i < ARRAY_SIZE(tsb_cache_names); i++) { unsigned long size = 8192 << i; const char *name = tsb_cache_names[i]; @@ -307,7 +385,7 @@ static unsigned long tsb_size_to_rss_limit(unsigned long new_size) * will not trigger any longer. * * The TSB can be anywhere from 8K to 1MB in size, in increasing powers - * of two. The TSB must be aligned to it's size, so f.e. a 512K TSB + * of two. The TSB must be aligned to its size, so f.e. a 512K TSB * must be 512K aligned. It also must be physically contiguous, so we * cannot use vmalloc(). * @@ -324,8 +402,8 @@ void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long rss) unsigned long new_rss_limit; gfp_t gfp_flags; - if (max_tsb_size > (PAGE_SIZE << MAX_ORDER)) - max_tsb_size = (PAGE_SIZE << MAX_ORDER); + if (max_tsb_size > PAGE_SIZE << MAX_PAGE_ORDER) + max_tsb_size = PAGE_SIZE << MAX_PAGE_ORDER; new_cache_index = 0; for (new_size = 8192; new_size < max_tsb_size; new_size <<= 1UL) { @@ -419,7 +497,8 @@ retry_tsb_alloc: extern void copy_tsb(unsigned long old_tsb_base, unsigned long old_tsb_size, unsigned long new_tsb_base, - unsigned long new_tsb_size); + unsigned long new_tsb_size, + unsigned long page_size_shift); unsigned long old_tsb_base = (unsigned long) old_tsb; unsigned long new_tsb_base = (unsigned long) new_tsb; @@ -427,7 +506,9 @@ retry_tsb_alloc: old_tsb_base = __pa(old_tsb_base); new_tsb_base = __pa(new_tsb_base); } - copy_tsb(old_tsb_base, old_size, new_tsb_base, new_size); + copy_tsb(old_tsb_base, old_size, new_tsb_base, new_size, + tsb_index == MM_TSB_BASE ? + PAGE_SHIFT : REAL_HPAGE_SHIFT); } mm->context.tsb_block[tsb_index].tsb = new_tsb; @@ -454,8 +535,10 @@ retry_tsb_alloc: int init_new_context(struct task_struct *tsk, struct mm_struct *mm) { + unsigned long mm_rss = get_mm_rss(mm); #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - unsigned long huge_pte_count; + unsigned long saved_hugetlb_pte_count; + unsigned long saved_thp_pte_count; #endif unsigned int i; @@ -463,16 +546,21 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) mm->context.sparc64_ctx_val = 0UL; + mm->context.tag_store = NULL; + spin_lock_init(&mm->context.tag_lock); + #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - /* We reset it to zero because the fork() page copying + /* We reset them to zero because the fork() page copying * will re-increment the counters as the parent PTEs are * copied into the child address space. */ - huge_pte_count = mm->context.huge_pte_count; - mm->context.huge_pte_count = 0; -#endif + saved_hugetlb_pte_count = mm->context.hugetlb_pte_count; + saved_thp_pte_count = mm->context.thp_pte_count; + mm->context.hugetlb_pte_count = 0; + mm->context.thp_pte_count = 0; - mm->context.pgtable_page = NULL; + mm_rss -= saved_thp_pte_count * (HPAGE_SIZE / PAGE_SIZE); +#endif /* copy_mm() copies over the parent's mm_struct before calling * us, so we need to zero out the TSB pointer or else tsb_grow() @@ -484,11 +572,13 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) /* If this is fork, inherit the parent's TSB size. We would * grow it to that size on the first page fault anyways. */ - tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm)); + tsb_grow(mm, MM_TSB_BASE, mm_rss); #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (unlikely(huge_pte_count)) - tsb_grow(mm, MM_TSB_HUGE, huge_pte_count); + if (unlikely(saved_hugetlb_pte_count + saved_thp_pte_count)) + tsb_grow(mm, MM_TSB_HUGE, + (saved_hugetlb_pte_count + saved_thp_pte_count) * + REAL_HPAGE_PER_HPAGE); #endif if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb)) @@ -512,17 +602,10 @@ static void tsb_destroy_one(struct tsb_config *tp) void destroy_context(struct mm_struct *mm) { unsigned long flags, i; - struct page *page; for (i = 0; i < MM_NUM_TSBS; i++) tsb_destroy_one(&mm->context.tsb_block[i]); - page = mm->context.pgtable_page; - if (page && put_page_testzero(page)) { - pgtable_page_dtor(page); - free_hot_cold_page(page, 0); - } - spin_lock_irqsave(&ctx_alloc_lock, flags); if (CTX_VALID(mm->context)) { @@ -531,4 +614,22 @@ void destroy_context(struct mm_struct *mm) } spin_unlock_irqrestore(&ctx_alloc_lock, flags); + + /* If ADI tag storage was allocated for this task, free it */ + if (mm->context.tag_store) { + tag_storage_desc_t *tag_desc; + unsigned long max_desc; + unsigned char *tags; + + tag_desc = mm->context.tag_store; + max_desc = PAGE_SIZE/sizeof(tag_storage_desc_t); + for (i = 0; i < max_desc; i++) { + tags = tag_desc->tags; + tag_desc->tags = NULL; + kfree(tags); + tag_desc++; + } + kfree(mm->context.tag_store); + mm->context.tag_store = NULL; + } } diff --git a/arch/sparc/mm/tsunami.S b/arch/sparc/mm/tsunami.S index bf10a345fa8b..62b742df65dc 100644 --- a/arch/sparc/mm/tsunami.S +++ b/arch/sparc/mm/tsunami.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * tsunami.S: High speed MicroSparc-I mmu/cache operations. * diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S index 432aa0cb1b38..70e658d107e0 100644 --- a/arch/sparc/mm/ultra.S +++ b/arch/sparc/mm/ultra.S @@ -1,11 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * ultra.S: Don't expand these all over the place... * * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net) */ +#include <linux/pgtable.h> #include <asm/asi.h> -#include <asm/pgtable.h> #include <asm/page.h> #include <asm/spitfire.h> #include <asm/mmu_context.h> @@ -30,7 +31,7 @@ .text .align 32 .globl __flush_tlb_mm -__flush_tlb_mm: /* 18 insns */ +__flush_tlb_mm: /* 19 insns */ /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ ldxa [%o1] ASI_DMMU, %g2 cmp %g2, %o0 @@ -81,7 +82,7 @@ __flush_tlb_page: /* 22 insns */ .align 32 .globl __flush_tlb_pending -__flush_tlb_pending: /* 26 insns */ +__flush_tlb_pending: /* 27 insns */ /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ rdpr %pstate, %g7 sllx %o1, 3, %o1 @@ -113,12 +114,14 @@ __flush_tlb_pending: /* 26 insns */ .align 32 .globl __flush_tlb_kernel_range -__flush_tlb_kernel_range: /* 16 insns */ +__flush_tlb_kernel_range: /* 31 insns */ /* %o0=start, %o1=end */ cmp %o0, %o1 be,pn %xcc, 2f + sub %o1, %o0, %o3 + srlx %o3, 18, %o4 + brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow sethi %hi(PAGE_SIZE), %o4 - sub %o1, %o0, %o3 sub %o3, %o4, %o3 or %o0, 0x20, %o0 ! Nucleus 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP @@ -131,6 +134,41 @@ __flush_tlb_kernel_range: /* 16 insns */ retl nop nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + +__spitfire_flush_tlb_kernel_range_slow: + mov 63 * 8, %o4 +1: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3 + andcc %o3, 0x40, %g0 /* _PAGE_L_4U */ + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %o3 + stxa %g0, [%o3] ASI_IMMU + stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS + membar #Sync +2: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3 + andcc %o3, 0x40, %g0 + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %o3 + stxa %g0, [%o3] ASI_DMMU + stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS + membar #Sync +2: sub %o4, 8, %o4 + brgez,pt %o4, 1b + nop + retl + nop __spitfire_flush_tlb_mm_slow: rdpr %pstate, %g1 @@ -153,10 +191,10 @@ __spitfire_flush_tlb_mm_slow: .globl __flush_icache_page __flush_icache_page: /* %o0 = phys_page */ srlx %o0, PAGE_SHIFT, %o0 - sethi %uhi(PAGE_OFFSET), %g1 + sethi %hi(PAGE_OFFSET), %g1 sllx %o0, PAGE_SHIFT, %o0 sethi %hi(PAGE_SIZE), %g2 - sllx %g1, 32, %g1 + ldx [%g1 + %lo(PAGE_OFFSET)], %g1 add %o0, %g1, %o0 1: subcc %g2, 32, %g2 bne,pt %icc, 1b @@ -178,8 +216,8 @@ __flush_icache_page: /* %o0 = phys_page */ .align 64 .globl __flush_dcache_page __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ - sethi %uhi(PAGE_OFFSET), %g1 - sllx %g1, 32, %g1 + sethi %hi(PAGE_OFFSET), %g1 + ldx [%g1 + %lo(PAGE_OFFSET)], %g1 sub %o0, %g1, %o0 ! physical address srlx %o0, 11, %o0 ! make D-cache TAG sethi %hi(1 << 14), %o2 ! D-cache size @@ -285,10 +323,44 @@ __cheetah_flush_tlb_pending: /* 27 insns */ retl wrpr %g7, 0x0, %pstate +__cheetah_flush_tlb_kernel_range: /* 31 insns */ + /* %o0=start, %o1=end */ + cmp %o0, %o1 + be,pn %xcc, 2f + sub %o1, %o0, %o3 + srlx %o3, 18, %o4 + brnz,pn %o4, 3f + sethi %hi(PAGE_SIZE), %o4 + sub %o3, %o4, %o3 + or %o0, 0x20, %o0 ! Nucleus +1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP + stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP + membar #Sync + brnz,pt %o3, 1b + sub %o3, %o4, %o3 +2: sethi %hi(KERNBASE), %o3 + flush %o3 + retl + nop +3: mov 0x80, %o4 + stxa %g0, [%o4] ASI_DMMU_DEMAP + membar #Sync + stxa %g0, [%o4] ASI_IMMU_DEMAP + membar #Sync + retl + nop + nop + nop + nop + nop + nop + nop + nop + #ifdef DCACHE_ALIASING_POSSIBLE __cheetah_flush_dcache_page: /* 11 insns */ - sethi %uhi(PAGE_OFFSET), %g1 - sllx %g1, 32, %g1 + sethi %hi(PAGE_OFFSET), %g1 + ldx [%g1 + %lo(PAGE_OFFSET)], %g1 sub %o0, %g1, %o0 sethi %hi(PAGE_SIZE), %o4 1: subcc %o4, (1 << 5), %o4 @@ -309,19 +381,28 @@ __hypervisor_tlb_tl0_error: ret restore -__hypervisor_flush_tlb_mm: /* 10 insns */ +__hypervisor_flush_tlb_mm: /* 19 insns */ mov %o0, %o2 /* ARG2: mmu context */ mov 0, %o0 /* ARG0: CPU lists unimplemented */ mov 0, %o1 /* ARG1: CPU lists unimplemented */ mov HV_MMU_ALL, %o3 /* ARG3: flags */ mov HV_FAST_MMU_DEMAP_CTX, %o5 ta HV_FAST_TRAP - brnz,pn %o0, __hypervisor_tlb_tl0_error + brnz,pn %o0, 1f mov HV_FAST_MMU_DEMAP_CTX, %o1 retl nop +1: sethi %hi(__hypervisor_tlb_tl0_error), %o5 + jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop + nop + nop + nop + nop + nop + nop -__hypervisor_flush_tlb_page: /* 11 insns */ +__hypervisor_flush_tlb_page: /* 22 insns */ /* %o0 = context, %o1 = vaddr */ mov %o0, %g2 mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ @@ -330,12 +411,23 @@ __hypervisor_flush_tlb_page: /* 11 insns */ srlx %o0, PAGE_SHIFT, %o0 sllx %o0, PAGE_SHIFT, %o0 ta HV_MMU_UNMAP_ADDR_TRAP - brnz,pn %o0, __hypervisor_tlb_tl0_error + brnz,pn %o0, 1f mov HV_MMU_UNMAP_ADDR_TRAP, %o1 retl nop +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop + nop + nop + nop + nop + nop + nop + nop + nop -__hypervisor_flush_tlb_pending: /* 16 insns */ +__hypervisor_flush_tlb_pending: /* 27 insns */ /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ sllx %o1, 3, %g1 mov %o2, %g2 @@ -347,31 +439,57 @@ __hypervisor_flush_tlb_pending: /* 16 insns */ srlx %o0, PAGE_SHIFT, %o0 sllx %o0, PAGE_SHIFT, %o0 ta HV_MMU_UNMAP_ADDR_TRAP - brnz,pn %o0, __hypervisor_tlb_tl0_error + brnz,pn %o0, 1f mov HV_MMU_UNMAP_ADDR_TRAP, %o1 brnz,pt %g1, 1b nop retl nop +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop + nop + nop + nop + nop + nop + nop + nop + nop -__hypervisor_flush_tlb_kernel_range: /* 16 insns */ +__hypervisor_flush_tlb_kernel_range: /* 31 insns */ /* %o0=start, %o1=end */ cmp %o0, %o1 be,pn %xcc, 2f - sethi %hi(PAGE_SIZE), %g3 - mov %o0, %g1 - sub %o1, %g1, %g2 + sub %o1, %o0, %g2 + srlx %g2, 18, %g3 + brnz,pn %g3, 4f + mov %o0, %g1 + sethi %hi(PAGE_SIZE), %g3 sub %g2, %g3, %g2 1: add %g1, %g2, %o0 /* ARG0: virtual address */ mov 0, %o1 /* ARG1: mmu context */ mov HV_MMU_ALL, %o2 /* ARG2: flags */ ta HV_MMU_UNMAP_ADDR_TRAP - brnz,pn %o0, __hypervisor_tlb_tl0_error + brnz,pn %o0, 3f mov HV_MMU_UNMAP_ADDR_TRAP, %o1 brnz,pt %g2, 1b sub %g2, %g3, %g2 2: retl nop +3: sethi %hi(__hypervisor_tlb_tl0_error), %o2 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop +4: mov 0, %o0 /* ARG0: CPU lists unimplemented */ + mov 0, %o1 /* ARG1: CPU lists unimplemented */ + mov 0, %o2 /* ARG2: mmu context == nucleus */ + mov HV_MMU_ALL, %o3 /* ARG3: flags */ + mov HV_FAST_MMU_DEMAP_CTX, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, 3b + mov HV_FAST_MMU_DEMAP_CTX, %o1 + retl + nop #ifdef DCACHE_ALIASING_POSSIBLE /* XXX Niagara and friends have an 8K cache, so no aliasing is @@ -394,43 +512,6 @@ tlb_patch_one: retl nop - .globl cheetah_patch_cachetlbops -cheetah_patch_cachetlbops: - save %sp, -128, %sp - - sethi %hi(__flush_tlb_mm), %o0 - or %o0, %lo(__flush_tlb_mm), %o0 - sethi %hi(__cheetah_flush_tlb_mm), %o1 - or %o1, %lo(__cheetah_flush_tlb_mm), %o1 - call tlb_patch_one - mov 19, %o2 - - sethi %hi(__flush_tlb_page), %o0 - or %o0, %lo(__flush_tlb_page), %o0 - sethi %hi(__cheetah_flush_tlb_page), %o1 - or %o1, %lo(__cheetah_flush_tlb_page), %o1 - call tlb_patch_one - mov 22, %o2 - - sethi %hi(__flush_tlb_pending), %o0 - or %o0, %lo(__flush_tlb_pending), %o0 - sethi %hi(__cheetah_flush_tlb_pending), %o1 - or %o1, %lo(__cheetah_flush_tlb_pending), %o1 - call tlb_patch_one - mov 27, %o2 - -#ifdef DCACHE_ALIASING_POSSIBLE - sethi %hi(__flush_dcache_page), %o0 - or %o0, %lo(__flush_dcache_page), %o0 - sethi %hi(__cheetah_flush_dcache_page), %o1 - or %o1, %lo(__cheetah_flush_dcache_page), %o1 - call tlb_patch_one - mov 11, %o2 -#endif /* DCACHE_ALIASING_POSSIBLE */ - - ret - restore - #ifdef CONFIG_SMP /* These are all called by the slaves of a cross call, at * trap level 1, with interrupts fully disabled. @@ -447,7 +528,7 @@ cheetah_patch_cachetlbops: */ .align 32 .globl xcall_flush_tlb_mm -xcall_flush_tlb_mm: /* 21 insns */ +xcall_flush_tlb_mm: /* 24 insns */ mov PRIMARY_CONTEXT, %g2 ldxa [%g2] ASI_DMMU, %g3 srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 @@ -469,9 +550,12 @@ xcall_flush_tlb_mm: /* 21 insns */ nop nop nop + nop + nop + nop .globl xcall_flush_tlb_page -xcall_flush_tlb_page: /* 17 insns */ +xcall_flush_tlb_page: /* 20 insns */ /* %g5=context, %g1=vaddr */ mov PRIMARY_CONTEXT, %g4 ldxa [%g4] ASI_DMMU, %g2 @@ -490,15 +574,20 @@ xcall_flush_tlb_page: /* 17 insns */ retry nop nop + nop + nop + nop .globl xcall_flush_tlb_kernel_range -xcall_flush_tlb_kernel_range: /* 25 insns */ +xcall_flush_tlb_kernel_range: /* 44 insns */ sethi %hi(PAGE_SIZE - 1), %g2 or %g2, %lo(PAGE_SIZE - 1), %g2 andn %g1, %g2, %g1 andn %g7, %g2, %g7 sub %g7, %g1, %g3 - add %g2, 1, %g2 + srlx %g3, 18, %g2 + brnz,pn %g2, 2f + sethi %hi(PAGE_SIZE), %g2 sub %g3, %g2, %g3 or %g1, 0x20, %g1 ! Nucleus 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP @@ -507,8 +596,25 @@ xcall_flush_tlb_kernel_range: /* 25 insns */ brnz,pt %g3, 1b sub %g3, %g2, %g3 retry - nop - nop +2: mov 63 * 8, %g1 +1: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2 + andcc %g2, 0x40, %g0 /* _PAGE_L_4U */ + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %g2 + stxa %g0, [%g2] ASI_IMMU + stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS + membar #Sync +2: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2 + andcc %g2, 0x40, %g0 + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %g2 + stxa %g0, [%g2] ASI_DMMU + stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS + membar #Sync +2: sub %g1, 8, %g1 + brgez,pt %g1, 1b + nop + retry nop nop nop @@ -637,6 +743,52 @@ xcall_fetch_glob_pmu_n4: retry +__cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */ + sethi %hi(PAGE_SIZE - 1), %g2 + or %g2, %lo(PAGE_SIZE - 1), %g2 + andn %g1, %g2, %g1 + andn %g7, %g2, %g7 + sub %g7, %g1, %g3 + srlx %g3, 18, %g2 + brnz,pn %g2, 2f + sethi %hi(PAGE_SIZE), %g2 + sub %g3, %g2, %g3 + or %g1, 0x20, %g1 ! Nucleus +1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP + stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP + membar #Sync + brnz,pt %g3, 1b + sub %g3, %g2, %g3 + retry +2: mov 0x80, %g2 + stxa %g0, [%g2] ASI_DMMU_DEMAP + membar #Sync + stxa %g0, [%g2] ASI_IMMU_DEMAP + membar #Sync + retry + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + #ifdef DCACHE_ALIASING_POSSIBLE .align 32 .globl xcall_flush_dcache_page_cheetah @@ -700,7 +852,7 @@ __hypervisor_tlb_xcall_error: ba,a,pt %xcc, rtrap .globl __hypervisor_xcall_flush_tlb_mm -__hypervisor_xcall_flush_tlb_mm: /* 21 insns */ +__hypervisor_xcall_flush_tlb_mm: /* 24 insns */ /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ mov %o0, %g2 mov %o1, %g3 @@ -714,7 +866,7 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ mov HV_FAST_MMU_DEMAP_CTX, %o5 ta HV_FAST_TRAP mov HV_FAST_MMU_DEMAP_CTX, %g6 - brnz,pn %o0, __hypervisor_tlb_xcall_error + brnz,pn %o0, 1f mov %o0, %g5 mov %g2, %o0 mov %g3, %o1 @@ -723,9 +875,12 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ mov %g7, %o5 membar #Sync retry +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 + nop .globl __hypervisor_xcall_flush_tlb_page -__hypervisor_xcall_flush_tlb_page: /* 17 insns */ +__hypervisor_xcall_flush_tlb_page: /* 20 insns */ /* %g5=ctx, %g1=vaddr */ mov %o0, %g2 mov %o1, %g3 @@ -737,42 +892,64 @@ __hypervisor_xcall_flush_tlb_page: /* 17 insns */ sllx %o0, PAGE_SHIFT, %o0 ta HV_MMU_UNMAP_ADDR_TRAP mov HV_MMU_UNMAP_ADDR_TRAP, %g6 - brnz,a,pn %o0, __hypervisor_tlb_xcall_error + brnz,a,pn %o0, 1f mov %o0, %g5 mov %g2, %o0 mov %g3, %o1 mov %g4, %o2 membar #Sync retry +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 + nop .globl __hypervisor_xcall_flush_tlb_kernel_range -__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */ +__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */ /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ sethi %hi(PAGE_SIZE - 1), %g2 or %g2, %lo(PAGE_SIZE - 1), %g2 andn %g1, %g2, %g1 andn %g7, %g2, %g7 sub %g7, %g1, %g3 + srlx %g3, 18, %g7 add %g2, 1, %g2 sub %g3, %g2, %g3 mov %o0, %g2 mov %o1, %g4 - mov %o2, %g7 + brnz,pn %g7, 2f + mov %o2, %g7 1: add %g1, %g3, %o0 /* ARG0: virtual address */ mov 0, %o1 /* ARG1: mmu context */ mov HV_MMU_ALL, %o2 /* ARG2: flags */ ta HV_MMU_UNMAP_ADDR_TRAP mov HV_MMU_UNMAP_ADDR_TRAP, %g6 - brnz,pn %o0, __hypervisor_tlb_xcall_error + brnz,pn %o0, 1f mov %o0, %g5 sethi %hi(PAGE_SIZE), %o2 brnz,pt %g3, 1b sub %g3, %o2, %g3 - mov %g2, %o0 +5: mov %g2, %o0 mov %g4, %o1 mov %g7, %o2 membar #Sync retry +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 + nop +2: mov %o3, %g1 + mov %o5, %g3 + mov 0, %o0 /* ARG0: CPU lists unimplemented */ + mov 0, %o1 /* ARG1: CPU lists unimplemented */ + mov 0, %o2 /* ARG2: mmu context == nucleus */ + mov HV_MMU_ALL, %o3 /* ARG3: flags */ + mov HV_FAST_MMU_DEMAP_CTX, %o5 + ta HV_FAST_TRAP + mov %g1, %o3 + brz,pt %o0, 5b + mov %g3, %o5 + mov HV_FAST_MMU_DEMAP_CTX, %g6 + ba,pt %xcc, 1b + clr %g5 /* These just get rescheduled to PIL vectors. */ .globl xcall_call_function @@ -795,11 +972,6 @@ xcall_capture: wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint retry - .globl xcall_new_mmu_context_version -xcall_new_mmu_context_version: - wr %g0, (1 << PIL_SMP_CTX_NEW_VERSION), %set_softint - retry - #ifdef CONFIG_KGDB .globl xcall_kgdb_capture xcall_kgdb_capture: @@ -809,6 +981,58 @@ xcall_kgdb_capture: #endif /* CONFIG_SMP */ + .globl cheetah_patch_cachetlbops +cheetah_patch_cachetlbops: + save %sp, -128, %sp + + sethi %hi(__flush_tlb_mm), %o0 + or %o0, %lo(__flush_tlb_mm), %o0 + sethi %hi(__cheetah_flush_tlb_mm), %o1 + or %o1, %lo(__cheetah_flush_tlb_mm), %o1 + call tlb_patch_one + mov 19, %o2 + + sethi %hi(__flush_tlb_page), %o0 + or %o0, %lo(__flush_tlb_page), %o0 + sethi %hi(__cheetah_flush_tlb_page), %o1 + or %o1, %lo(__cheetah_flush_tlb_page), %o1 + call tlb_patch_one + mov 22, %o2 + + sethi %hi(__flush_tlb_pending), %o0 + or %o0, %lo(__flush_tlb_pending), %o0 + sethi %hi(__cheetah_flush_tlb_pending), %o1 + or %o1, %lo(__cheetah_flush_tlb_pending), %o1 + call tlb_patch_one + mov 27, %o2 + + sethi %hi(__flush_tlb_kernel_range), %o0 + or %o0, %lo(__flush_tlb_kernel_range), %o0 + sethi %hi(__cheetah_flush_tlb_kernel_range), %o1 + or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1 + call tlb_patch_one + mov 31, %o2 + +#ifdef DCACHE_ALIASING_POSSIBLE + sethi %hi(__flush_dcache_page), %o0 + or %o0, %lo(__flush_dcache_page), %o0 + sethi %hi(__cheetah_flush_dcache_page), %o1 + or %o1, %lo(__cheetah_flush_dcache_page), %o1 + call tlb_patch_one + mov 11, %o2 +#endif /* DCACHE_ALIASING_POSSIBLE */ + +#ifdef CONFIG_SMP + sethi %hi(xcall_flush_tlb_kernel_range), %o0 + or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 + sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1 + or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1 + call tlb_patch_one + mov 44, %o2 +#endif /* CONFIG_SMP */ + + ret + restore .globl hypervisor_patch_cachetlbops hypervisor_patch_cachetlbops: @@ -819,28 +1043,28 @@ hypervisor_patch_cachetlbops: sethi %hi(__hypervisor_flush_tlb_mm), %o1 or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 call tlb_patch_one - mov 10, %o2 + mov 19, %o2 sethi %hi(__flush_tlb_page), %o0 or %o0, %lo(__flush_tlb_page), %o0 sethi %hi(__hypervisor_flush_tlb_page), %o1 or %o1, %lo(__hypervisor_flush_tlb_page), %o1 call tlb_patch_one - mov 11, %o2 + mov 22, %o2 sethi %hi(__flush_tlb_pending), %o0 or %o0, %lo(__flush_tlb_pending), %o0 sethi %hi(__hypervisor_flush_tlb_pending), %o1 or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 call tlb_patch_one - mov 16, %o2 + mov 27, %o2 sethi %hi(__flush_tlb_kernel_range), %o0 or %o0, %lo(__flush_tlb_kernel_range), %o0 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 call tlb_patch_one - mov 16, %o2 + mov 31, %o2 #ifdef DCACHE_ALIASING_POSSIBLE sethi %hi(__flush_dcache_page), %o0 @@ -857,21 +1081,21 @@ hypervisor_patch_cachetlbops: sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 call tlb_patch_one - mov 21, %o2 + mov 24, %o2 sethi %hi(xcall_flush_tlb_page), %o0 or %o0, %lo(xcall_flush_tlb_page), %o0 sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 call tlb_patch_one - mov 17, %o2 + mov 20, %o2 sethi %hi(xcall_flush_tlb_kernel_range), %o0 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 call tlb_patch_one - mov 25, %o2 + mov 44, %o2 #endif /* CONFIG_SMP */ ret diff --git a/arch/sparc/mm/viking.S b/arch/sparc/mm/viking.S index 852257fcc82b..48f062de7a7f 100644 --- a/arch/sparc/mm/viking.S +++ b/arch/sparc/mm/viking.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * viking.S: High speed Viking cache/mmu operations * @@ -12,6 +13,7 @@ #include <asm/asi.h> #include <asm/mxcc.h> #include <asm/page.h> +#include <asm/pgtable.h> #include <asm/pgtsrmmu.h> #include <asm/viking.h> @@ -156,7 +158,7 @@ viking_flush_tlb_range: cmp %o3, -1 be 2f #endif - sethi %hi(~((1 << SRMMU_PGDIR_SHIFT) - 1)), %o4 + sethi %hi(~((1 << PGDIR_SHIFT) - 1)), %o4 sta %o3, [%g1] ASI_M_MMUREGS and %o1, %o4, %o1 add %o1, 0x200, %o1 @@ -242,7 +244,7 @@ sun4dsmp_flush_tlb_range: ld [%o0 + VMA_VM_MM], %o0 ld [%o0 + AOFF_mm_context], %o3 lda [%g1] ASI_M_MMUREGS, %g5 - sethi %hi(~((1 << SRMMU_PGDIR_SHIFT) - 1)), %o4 + sethi %hi(~((1 << PGDIR_SHIFT) - 1)), %o4 sta %o3, [%g1] ASI_M_MMUREGS and %o1, %o4, %o1 add %o1, 0x200, %o1 diff --git a/arch/sparc/net/Makefile b/arch/sparc/net/Makefile index 1306a58ac541..806267de3bdf 100644 --- a/arch/sparc/net/Makefile +++ b/arch/sparc/net/Makefile @@ -1,4 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only # # Arch-specific network modules # -obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o +obj-$(CONFIG_BPF_JIT) += bpf_jit_comp_$(BITS).o +ifeq ($(BITS),32) +obj-$(CONFIG_BPF_JIT) += bpf_jit_asm_32.o +endif diff --git a/arch/sparc/net/bpf_jit.h b/arch/sparc/net/bpf_jit_32.h index 33d6b375ff12..cfd6a8be0eb4 100644 --- a/arch/sparc/net/bpf_jit.h +++ b/arch/sparc/net/bpf_jit_32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _BPF_JIT_H #define _BPF_JIT_H @@ -39,7 +40,7 @@ #define r_TMP2 G2 #define r_OFF G3 -/* assembly code in arch/sparc/net/bpf_jit_asm.S */ +/* assembly code in arch/sparc/net/bpf_jit_asm_32.S */ extern u32 bpf_jit_load_word[]; extern u32 bpf_jit_load_half[]; extern u32 bpf_jit_load_byte[]; diff --git a/arch/sparc/net/bpf_jit_64.h b/arch/sparc/net/bpf_jit_64.h new file mode 100644 index 000000000000..fbc836f1c51c --- /dev/null +++ b/arch/sparc/net/bpf_jit_64.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BPF_JIT_H +#define _BPF_JIT_H + +#ifndef __ASSEMBLER__ +#define G0 0x00 +#define G1 0x01 +#define G2 0x02 +#define G3 0x03 +#define G6 0x06 +#define G7 0x07 +#define O0 0x08 +#define O1 0x09 +#define O2 0x0a +#define O3 0x0b +#define O4 0x0c +#define O5 0x0d +#define SP 0x0e +#define O7 0x0f +#define L0 0x10 +#define L1 0x11 +#define L2 0x12 +#define L3 0x13 +#define L4 0x14 +#define L5 0x15 +#define L6 0x16 +#define L7 0x17 +#define I0 0x18 +#define I1 0x19 +#define I2 0x1a +#define I3 0x1b +#define I4 0x1c +#define I5 0x1d +#define FP 0x1e +#define I7 0x1f +#endif + +#endif /* _BPF_JIT_H */ diff --git a/arch/sparc/net/bpf_jit_asm.S b/arch/sparc/net/bpf_jit_asm_32.S index 9d016c7017f7..a2e28e0464a3 100644 --- a/arch/sparc/net/bpf_jit_asm.S +++ b/arch/sparc/net/bpf_jit_asm_32.S @@ -1,16 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <asm/ptrace.h> -#include "bpf_jit.h" +#include "bpf_jit_32.h" -#ifdef CONFIG_SPARC64 -#define SAVE_SZ 176 -#define SCRATCH_OFF STACK_BIAS + 128 -#define BE_PTR(label) be,pn %xcc, label -#else #define SAVE_SZ 96 #define SCRATCH_OFF 72 #define BE_PTR(label) be label -#endif +#define SIGN_EXTEND(reg) #define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */ @@ -135,6 +131,7 @@ bpf_slow_path_byte_msh: save %sp, -SAVE_SZ, %sp; \ mov %i0, %o0; \ mov r_OFF, %o1; \ + SIGN_EXTEND(%o1); \ call bpf_internal_load_pointer_neg_helper; \ mov (LEN), %o2; \ mov %o0, r_TMP; \ diff --git a/arch/sparc/net/bpf_jit_comp.c b/arch/sparc/net/bpf_jit_comp_32.c index 9c7be59e6f5a..bda2dbd3f4c5 100644 --- a/arch/sparc/net/bpf_jit_comp.c +++ b/arch/sparc/net/bpf_jit_comp_32.c @@ -1,40 +1,21 @@ -#include <linux/moduleloader.h> +// SPDX-License-Identifier: GPL-2.0 #include <linux/workqueue.h> #include <linux/netdevice.h> #include <linux/filter.h> #include <linux/cache.h> #include <linux/if_vlan.h> +#include <linux/execmem.h> #include <asm/cacheflush.h> #include <asm/ptrace.h> -#include "bpf_jit.h" - -int bpf_jit_enable __read_mostly; +#include "bpf_jit_32.h" static inline bool is_simm13(unsigned int value) { return value + 0x1000 < 0x2000; } -static void bpf_flush_icache(void *start_, void *end_) -{ -#ifdef CONFIG_SPARC64 - /* Cheetah's I-cache is fully coherent. */ - if (tlb_type == spitfire) { - unsigned long start = (unsigned long) start_; - unsigned long end = (unsigned long) end_; - - start &= ~7UL; - end = (end + 7UL) & ~7UL; - while (start < end) { - flushi(start); - start += 32; - } - } -#endif -} - #define SEEN_DATAREF 1 /* might call external helpers */ #define SEEN_XREG 2 /* ebx is used */ #define SEEN_MEM 4 /* use mem[] for temporary storage */ @@ -82,11 +63,7 @@ static void bpf_flush_icache(void *start_, void *end_) #define BE (F2(0, 2) | CONDE) #define BNE (F2(0, 2) | CONDNE) -#ifdef CONFIG_SPARC64 -#define BNE_PTR (F2(0, 1) | CONDNE | (2 << 20)) -#else -#define BNE_PTR BNE -#endif +#define BE_PTR BE #define SETHI(K, REG) \ (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff)) @@ -116,13 +93,8 @@ static void bpf_flush_icache(void *start_, void *end_) #define LD64 F3(3, 0x0b) #define ST32 F3(3, 0x04) -#ifdef CONFIG_SPARC64 -#define LDPTR LD64 -#define BASE_STACKFRAME 176 -#else #define LDPTR LD32 #define BASE_STACKFRAME 96 -#endif #define LD32I (LD32 | IMMED) #define LD8I (LD8 | IMMED) @@ -184,7 +156,7 @@ do { \ */ #define emit_alu_K(OPCODE, K) \ do { \ - if (K) { \ + if (K || OPCODE == AND || OPCODE == MUL) { \ unsigned int _insn = OPCODE; \ _insn |= RS1(r_A) | RD(r_A); \ if (is_simm13(K)) { \ @@ -208,19 +180,19 @@ do { \ #define emit_loadptr(BASE, STRUCT, FIELD, DEST) \ do { unsigned int _off = offsetof(STRUCT, FIELD); \ - BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(void *)); \ + BUILD_BUG_ON(sizeof_field(STRUCT, FIELD) != sizeof(void *)); \ *prog++ = LDPTRI | RS1(BASE) | S13(_off) | RD(DEST); \ } while (0) #define emit_load32(BASE, STRUCT, FIELD, DEST) \ do { unsigned int _off = offsetof(STRUCT, FIELD); \ - BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(u32)); \ + BUILD_BUG_ON(sizeof_field(STRUCT, FIELD) != sizeof(u32)); \ *prog++ = LD32I | RS1(BASE) | S13(_off) | RD(DEST); \ } while (0) #define emit_load16(BASE, STRUCT, FIELD, DEST) \ do { unsigned int _off = offsetof(STRUCT, FIELD); \ - BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(u16)); \ + BUILD_BUG_ON(sizeof_field(STRUCT, FIELD) != sizeof(u16)); \ *prog++ = LD16I | RS1(BASE) | S13(_off) | RD(DEST); \ } while (0) @@ -230,26 +202,23 @@ do { unsigned int _off = offsetof(STRUCT, FIELD); \ } while (0) #define emit_load8(BASE, STRUCT, FIELD, DEST) \ -do { BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(u8)); \ +do { BUILD_BUG_ON(sizeof_field(STRUCT, FIELD) != sizeof(u8)); \ __emit_load8(BASE, STRUCT, FIELD, DEST); \ } while (0) -#define emit_ldmem(OFF, DEST) \ -do { *prog++ = LD32I | RS1(FP) | S13(-(OFF)) | RD(DEST); \ +#define BIAS (-4) + +#define emit_ldmem(OFF, DEST) \ +do { *prog++ = LD32I | RS1(SP) | S13(BIAS - (OFF)) | RD(DEST); \ } while (0) -#define emit_stmem(OFF, SRC) \ -do { *prog++ = LD32I | RS1(FP) | S13(-(OFF)) | RD(SRC); \ +#define emit_stmem(OFF, SRC) \ +do { *prog++ = ST32I | RS1(SP) | S13(BIAS - (OFF)) | RD(SRC); \ } while (0) #ifdef CONFIG_SMP -#ifdef CONFIG_SPARC64 -#define emit_load_cpu(REG) \ - emit_load16(G6, struct thread_info, cpu, REG) -#else #define emit_load_cpu(REG) \ emit_load32(G6, struct thread_info, cpu, REG) -#endif #else #define emit_load_cpu(REG) emit_clear(REG) #endif @@ -331,7 +300,7 @@ do { *prog++ = BR_OPC | WDISP22(OFF); \ * * The most common case is to emit a branch at the end of such * a code sequence. So this would be two instructions, the - * branch and it's delay slot. + * branch and its delay slot. * * Therefore by default the branch emitters calculate the branch * offset field as: @@ -340,13 +309,13 @@ do { *prog++ = BR_OPC | WDISP22(OFF); \ * * This "addrs[i] - 8" is the address of the branch itself or * what "." would be in assembler notation. The "8" part is - * how we take into consideration the branch and it's delay + * how we take into consideration the branch and its delay * slot mentioned above. * * Sometimes we need to emit a branch earlier in the code * sequence. And in these situations we adjust "destination" - * to accomodate this difference. For example, if we needed - * to emit a branch (and it's delay slot) right before the + * to accommodate this difference. For example, if we needed + * to emit a branch (and its delay slot) right before the * final instruction emitted for a BPF opcode, we'd use * "destination + 4" instead of just plain "destination" above. * @@ -354,7 +323,7 @@ do { *prog++ = BR_OPC | WDISP22(OFF); \ * emit_jump() calls with adjusted offsets. */ -void bpf_jit_compile(struct sk_filter *fp) +void bpf_jit_compile(struct bpf_prog *fp) { unsigned int cleanup_addr, proglen, oldproglen = 0; u32 temp[8], *prog, *func, seen = 0, pass; @@ -366,7 +335,7 @@ void bpf_jit_compile(struct sk_filter *fp) if (!bpf_jit_enable) return; - addrs = kmalloc(flen * sizeof(*addrs), GFP_KERNEL); + addrs = kmalloc_array(flen, sizeof(*addrs), GFP_KERNEL); if (addrs == NULL) return; @@ -414,115 +383,94 @@ void bpf_jit_compile(struct sk_filter *fp) } emit_reg_move(O7, r_saved_O7); - switch (filter[0].code) { - case BPF_S_RET_K: - case BPF_S_LD_W_LEN: - case BPF_S_ANC_PROTOCOL: - case BPF_S_ANC_PKTTYPE: - case BPF_S_ANC_IFINDEX: - case BPF_S_ANC_MARK: - case BPF_S_ANC_RXHASH: - case BPF_S_ANC_VLAN_TAG: - case BPF_S_ANC_VLAN_TAG_PRESENT: - case BPF_S_ANC_CPU: - case BPF_S_ANC_QUEUE: - case BPF_S_LD_W_ABS: - case BPF_S_LD_H_ABS: - case BPF_S_LD_B_ABS: - /* The first instruction sets the A register (or is - * a "RET 'constant'") - */ - break; - default: - /* Make sure we dont leak kernel information to the - * user. - */ + /* Make sure we dont leak kernel information to the user. */ + if (bpf_needs_clear_a(&filter[0])) emit_clear(r_A); /* A = 0 */ - } for (i = 0; i < flen; i++) { unsigned int K = filter[i].k; unsigned int t_offset; unsigned int f_offset; u32 t_op, f_op; + u16 code = bpf_anc_helper(&filter[i]); int ilen; - switch (filter[i].code) { - case BPF_S_ALU_ADD_X: /* A += X; */ + switch (code) { + case BPF_ALU | BPF_ADD | BPF_X: /* A += X; */ emit_alu_X(ADD); break; - case BPF_S_ALU_ADD_K: /* A += K; */ + case BPF_ALU | BPF_ADD | BPF_K: /* A += K; */ emit_alu_K(ADD, K); break; - case BPF_S_ALU_SUB_X: /* A -= X; */ + case BPF_ALU | BPF_SUB | BPF_X: /* A -= X; */ emit_alu_X(SUB); break; - case BPF_S_ALU_SUB_K: /* A -= K */ + case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */ emit_alu_K(SUB, K); break; - case BPF_S_ALU_AND_X: /* A &= X */ + case BPF_ALU | BPF_AND | BPF_X: /* A &= X */ emit_alu_X(AND); break; - case BPF_S_ALU_AND_K: /* A &= K */ + case BPF_ALU | BPF_AND | BPF_K: /* A &= K */ emit_alu_K(AND, K); break; - case BPF_S_ALU_OR_X: /* A |= X */ + case BPF_ALU | BPF_OR | BPF_X: /* A |= X */ emit_alu_X(OR); break; - case BPF_S_ALU_OR_K: /* A |= K */ + case BPF_ALU | BPF_OR | BPF_K: /* A |= K */ emit_alu_K(OR, K); break; - case BPF_S_ANC_ALU_XOR_X: /* A ^= X; */ - case BPF_S_ALU_XOR_X: + case BPF_ANC | SKF_AD_ALU_XOR_X: /* A ^= X; */ + case BPF_ALU | BPF_XOR | BPF_X: emit_alu_X(XOR); break; - case BPF_S_ALU_XOR_K: /* A ^= K */ + case BPF_ALU | BPF_XOR | BPF_K: /* A ^= K */ emit_alu_K(XOR, K); break; - case BPF_S_ALU_LSH_X: /* A <<= X */ + case BPF_ALU | BPF_LSH | BPF_X: /* A <<= X */ emit_alu_X(SLL); break; - case BPF_S_ALU_LSH_K: /* A <<= K */ + case BPF_ALU | BPF_LSH | BPF_K: /* A <<= K */ emit_alu_K(SLL, K); break; - case BPF_S_ALU_RSH_X: /* A >>= X */ + case BPF_ALU | BPF_RSH | BPF_X: /* A >>= X */ emit_alu_X(SRL); break; - case BPF_S_ALU_RSH_K: /* A >>= K */ + case BPF_ALU | BPF_RSH | BPF_K: /* A >>= K */ emit_alu_K(SRL, K); break; - case BPF_S_ALU_MUL_X: /* A *= X; */ + case BPF_ALU | BPF_MUL | BPF_X: /* A *= X; */ emit_alu_X(MUL); break; - case BPF_S_ALU_MUL_K: /* A *= K */ + case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */ emit_alu_K(MUL, K); break; - case BPF_S_ALU_DIV_K: /* A /= K */ - emit_alu_K(MUL, K); - emit_read_y(r_A); + case BPF_ALU | BPF_DIV | BPF_K: /* A /= K with K != 0*/ + if (K == 1) + break; + emit_write_y(G0); + /* The Sparc v8 architecture requires + * three instructions between a %y + * register write and the first use. + */ + emit_nop(); + emit_nop(); + emit_nop(); + emit_alu_K(DIV, K); break; - case BPF_S_ALU_DIV_X: /* A /= X; */ + case BPF_ALU | BPF_DIV | BPF_X: /* A /= X; */ emit_cmpi(r_X, 0); if (pc_ret0 > 0) { t_offset = addrs[pc_ret0 - 1]; -#ifdef CONFIG_SPARC32 emit_branch(BE, t_offset + 20); -#else - emit_branch(BE, t_offset + 8); -#endif emit_nop(); /* delay slot */ } else { emit_branch_off(BNE, 16); emit_nop(); -#ifdef CONFIG_SPARC32 emit_jump(cleanup_addr + 20); -#else - emit_jump(cleanup_addr + 8); -#endif emit_clear(r_A); } emit_write_y(G0); -#ifdef CONFIG_SPARC32 /* The Sparc v8 architecture requires * three instructions between a %y * register write and the first use. @@ -530,13 +478,12 @@ void bpf_jit_compile(struct sk_filter *fp) emit_nop(); emit_nop(); emit_nop(); -#endif emit_alu_X(DIV); break; - case BPF_S_ALU_NEG: + case BPF_ALU | BPF_NEG: emit_neg(); break; - case BPF_S_RET_K: + case BPF_RET | BPF_K: if (!K) { if (pc_ret0 == -1) pc_ret0 = i; @@ -544,8 +491,8 @@ void bpf_jit_compile(struct sk_filter *fp) } else { emit_loadimm(K, r_A); } - /* Fallthrough */ - case BPF_S_RET_A: + fallthrough; + case BPF_RET | BPF_A: if (seen_or_pass0) { if (i != flen - 1) { emit_jump(cleanup_addr); @@ -562,102 +509,106 @@ void bpf_jit_compile(struct sk_filter *fp) emit_jmpl(r_saved_O7, 8, G0); emit_reg_move(r_A, O0); /* delay slot */ break; - case BPF_S_MISC_TAX: + case BPF_MISC | BPF_TAX: seen |= SEEN_XREG; emit_reg_move(r_A, r_X); break; - case BPF_S_MISC_TXA: + case BPF_MISC | BPF_TXA: seen |= SEEN_XREG; emit_reg_move(r_X, r_A); break; - case BPF_S_ANC_CPU: + case BPF_ANC | SKF_AD_CPU: emit_load_cpu(r_A); break; - case BPF_S_ANC_PROTOCOL: + case BPF_ANC | SKF_AD_PROTOCOL: emit_skb_load16(protocol, r_A); break; -#if 0 - /* GCC won't let us take the address of - * a bit field even though we very much - * know what we are doing here. - */ - case BPF_S_ANC_PKTTYPE: - __emit_skb_load8(pkt_type, r_A); + case BPF_ANC | SKF_AD_PKTTYPE: + __emit_skb_load8(__pkt_type_offset, r_A); + emit_andi(r_A, PKT_TYPE_MAX, r_A); emit_alu_K(SRL, 5); break; -#endif - case BPF_S_ANC_IFINDEX: + case BPF_ANC | SKF_AD_IFINDEX: emit_skb_loadptr(dev, r_A); emit_cmpi(r_A, 0); - emit_branch(BNE_PTR, cleanup_addr + 4); + emit_branch(BE_PTR, cleanup_addr + 4); emit_nop(); emit_load32(r_A, struct net_device, ifindex, r_A); break; - case BPF_S_ANC_MARK: + case BPF_ANC | SKF_AD_MARK: emit_skb_load32(mark, r_A); break; - case BPF_S_ANC_QUEUE: + case BPF_ANC | SKF_AD_QUEUE: emit_skb_load16(queue_mapping, r_A); break; - case BPF_S_ANC_HATYPE: + case BPF_ANC | SKF_AD_HATYPE: emit_skb_loadptr(dev, r_A); emit_cmpi(r_A, 0); - emit_branch(BNE_PTR, cleanup_addr + 4); + emit_branch(BE_PTR, cleanup_addr + 4); emit_nop(); emit_load16(r_A, struct net_device, type, r_A); break; - case BPF_S_ANC_RXHASH: - emit_skb_load32(rxhash, r_A); + case BPF_ANC | SKF_AD_RXHASH: + emit_skb_load32(hash, r_A); break; - case BPF_S_ANC_VLAN_TAG: - case BPF_S_ANC_VLAN_TAG_PRESENT: + case BPF_ANC | SKF_AD_VLAN_TAG: emit_skb_load16(vlan_tci, r_A); - if (filter[i].code == BPF_S_ANC_VLAN_TAG) { - emit_andi(r_A, VLAN_VID_MASK, r_A); - } else { - emit_loadimm(VLAN_TAG_PRESENT, r_TMP); - emit_and(r_A, r_TMP, r_A); - } break; - - case BPF_S_LD_IMM: + case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: + emit_skb_load32(vlan_all, r_A); + emit_cmpi(r_A, 0); + emit_branch_off(BE, 12); + emit_nop(); + emit_loadimm(1, r_A); + break; + case BPF_LD | BPF_W | BPF_LEN: + emit_skb_load32(len, r_A); + break; + case BPF_LDX | BPF_W | BPF_LEN: + emit_skb_load32(len, r_X); + break; + case BPF_LD | BPF_IMM: emit_loadimm(K, r_A); break; - case BPF_S_LDX_IMM: + case BPF_LDX | BPF_IMM: emit_loadimm(K, r_X); break; - case BPF_S_LD_MEM: + case BPF_LD | BPF_MEM: + seen |= SEEN_MEM; emit_ldmem(K * 4, r_A); break; - case BPF_S_LDX_MEM: + case BPF_LDX | BPF_MEM: + seen |= SEEN_MEM | SEEN_XREG; emit_ldmem(K * 4, r_X); break; - case BPF_S_ST: + case BPF_ST: + seen |= SEEN_MEM; emit_stmem(K * 4, r_A); break; - case BPF_S_STX: + case BPF_STX: + seen |= SEEN_MEM | SEEN_XREG; emit_stmem(K * 4, r_X); break; #define CHOOSE_LOAD_FUNC(K, func) \ ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset) - case BPF_S_LD_W_ABS: + case BPF_LD | BPF_W | BPF_ABS: func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_word); common_load: seen |= SEEN_DATAREF; emit_loadimm(K, r_OFF); emit_call(func); break; - case BPF_S_LD_H_ABS: + case BPF_LD | BPF_H | BPF_ABS: func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_half); goto common_load; - case BPF_S_LD_B_ABS: + case BPF_LD | BPF_B | BPF_ABS: func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_byte); goto common_load; - case BPF_S_LDX_B_MSH: + case BPF_LDX | BPF_B | BPF_MSH: func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_byte_msh); goto common_load; - case BPF_S_LD_W_IND: + case BPF_LD | BPF_W | BPF_IND: func = bpf_jit_load_word; common_load_ind: seen |= SEEN_DATAREF | SEEN_XREG; if (K) { @@ -672,13 +623,13 @@ common_load_ind: seen |= SEEN_DATAREF | SEEN_XREG; } emit_call(func); break; - case BPF_S_LD_H_IND: + case BPF_LD | BPF_H | BPF_IND: func = bpf_jit_load_half; goto common_load_ind; - case BPF_S_LD_B_IND: + case BPF_LD | BPF_B | BPF_IND: func = bpf_jit_load_byte; goto common_load_ind; - case BPF_S_JMP_JA: + case BPF_JMP | BPF_JA: emit_jump(addrs[i + K]); emit_nop(); break; @@ -689,14 +640,14 @@ common_load_ind: seen |= SEEN_DATAREF | SEEN_XREG; f_op = FOP; \ goto cond_branch - COND_SEL(BPF_S_JMP_JGT_K, BGU, BLEU); - COND_SEL(BPF_S_JMP_JGE_K, BGEU, BLU); - COND_SEL(BPF_S_JMP_JEQ_K, BE, BNE); - COND_SEL(BPF_S_JMP_JSET_K, BNE, BE); - COND_SEL(BPF_S_JMP_JGT_X, BGU, BLEU); - COND_SEL(BPF_S_JMP_JGE_X, BGEU, BLU); - COND_SEL(BPF_S_JMP_JEQ_X, BE, BNE); - COND_SEL(BPF_S_JMP_JSET_X, BNE, BE); + COND_SEL(BPF_JMP | BPF_JGT | BPF_K, BGU, BLEU); + COND_SEL(BPF_JMP | BPF_JGE | BPF_K, BGEU, BLU); + COND_SEL(BPF_JMP | BPF_JEQ | BPF_K, BE, BNE); + COND_SEL(BPF_JMP | BPF_JSET | BPF_K, BNE, BE); + COND_SEL(BPF_JMP | BPF_JGT | BPF_X, BGU, BLEU); + COND_SEL(BPF_JMP | BPF_JGE | BPF_X, BGEU, BLU); + COND_SEL(BPF_JMP | BPF_JEQ | BPF_X, BE, BNE); + COND_SEL(BPF_JMP | BPF_JSET | BPF_X, BNE, BE); cond_branch: f_offset = addrs[i + filter[i].jf]; t_offset = addrs[i + filter[i].jt]; @@ -708,20 +659,20 @@ cond_branch: f_offset = addrs[i + filter[i].jf]; break; } - switch (filter[i].code) { - case BPF_S_JMP_JGT_X: - case BPF_S_JMP_JGE_X: - case BPF_S_JMP_JEQ_X: + switch (code) { + case BPF_JMP | BPF_JGT | BPF_X: + case BPF_JMP | BPF_JGE | BPF_X: + case BPF_JMP | BPF_JEQ | BPF_X: seen |= SEEN_XREG; emit_cmp(r_A, r_X); break; - case BPF_S_JMP_JSET_X: + case BPF_JMP | BPF_JSET | BPF_X: seen |= SEEN_XREG; emit_btst(r_A, r_X); break; - case BPF_S_JMP_JEQ_K: - case BPF_S_JMP_JGT_K: - case BPF_S_JMP_JGE_K: + case BPF_JMP | BPF_JEQ | BPF_K: + case BPF_JMP | BPF_JGT | BPF_K: + case BPF_JMP | BPF_JGE | BPF_K: if (is_simm13(K)) { emit_cmpi(r_A, K); } else { @@ -729,7 +680,7 @@ cond_branch: f_offset = addrs[i + filter[i].jf]; emit_cmp(r_A, r_TMP); } break; - case BPF_S_JMP_JSET_K: + case BPF_JMP | BPF_JSET | BPF_K: if (is_simm13(K)) { emit_btsti(r_A, K); } else { @@ -762,7 +713,7 @@ cond_branch: f_offset = addrs[i + filter[i].jf]; if (unlikely(proglen + ilen > oldproglen)) { pr_err("bpb_jit_compile fatal error\n"); kfree(addrs); - module_free(NULL, image); + execmem_free(image); return; } memcpy(image + proglen, temp, ilen); @@ -785,7 +736,7 @@ cond_branch: f_offset = addrs[i + filter[i].jf]; break; } if (proglen == oldproglen) { - image = module_alloc(proglen); + image = execmem_alloc(EXECMEM_BPF, proglen); if (!image) goto out; } @@ -793,19 +744,21 @@ cond_branch: f_offset = addrs[i + filter[i].jf]; } if (bpf_jit_enable > 1) - bpf_jit_dump(flen, proglen, pass, image); + bpf_jit_dump(flen, proglen, pass + 1, image); if (image) { - bpf_flush_icache(image, image + proglen); fp->bpf_func = (void *)image; + fp->jited = 1; } out: kfree(addrs); return; } -void bpf_jit_free(struct sk_filter *fp) +void bpf_jit_free(struct bpf_prog *fp) { - if (fp->bpf_func != sk_run_filter) - module_free(NULL, fp->bpf_func); + if (fp->jited) + execmem_free(fp->bpf_func); + + bpf_prog_unlock_free(fp); } diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c new file mode 100644 index 000000000000..73bf0aea8baf --- /dev/null +++ b/arch/sparc/net/bpf_jit_comp_64.c @@ -0,0 +1,1632 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/moduleloader.h> +#include <linux/workqueue.h> +#include <linux/netdevice.h> +#include <linux/filter.h> +#include <linux/bpf.h> +#include <linux/cache.h> +#include <linux/if_vlan.h> + +#include <asm/cacheflush.h> +#include <asm/ptrace.h> + +#include "bpf_jit_64.h" + +static inline bool is_simm13(unsigned int value) +{ + return value + 0x1000 < 0x2000; +} + +static inline bool is_simm10(unsigned int value) +{ + return value + 0x200 < 0x400; +} + +static inline bool is_simm5(unsigned int value) +{ + return value + 0x10 < 0x20; +} + +static inline bool is_sethi(unsigned int value) +{ + return (value & ~0x3fffff) == 0; +} + +static void bpf_flush_icache(void *start_, void *end_) +{ + /* Cheetah's I-cache is fully coherent. */ + if (tlb_type == spitfire) { + unsigned long start = (unsigned long) start_; + unsigned long end = (unsigned long) end_; + + start &= ~7UL; + end = (end + 7UL) & ~7UL; + while (start < end) { + flushi(start); + start += 32; + } + } +} + +#define S13(X) ((X) & 0x1fff) +#define S5(X) ((X) & 0x1f) +#define IMMED 0x00002000 +#define RD(X) ((X) << 25) +#define RS1(X) ((X) << 14) +#define RS2(X) ((X)) +#define OP(X) ((X) << 30) +#define OP2(X) ((X) << 22) +#define OP3(X) ((X) << 19) +#define COND(X) (((X) & 0xf) << 25) +#define CBCOND(X) (((X) & 0x1f) << 25) +#define F1(X) OP(X) +#define F2(X, Y) (OP(X) | OP2(Y)) +#define F3(X, Y) (OP(X) | OP3(Y)) +#define ASI(X) (((X) & 0xff) << 5) + +#define CONDN COND(0x0) +#define CONDE COND(0x1) +#define CONDLE COND(0x2) +#define CONDL COND(0x3) +#define CONDLEU COND(0x4) +#define CONDCS COND(0x5) +#define CONDNEG COND(0x6) +#define CONDVC COND(0x7) +#define CONDA COND(0x8) +#define CONDNE COND(0x9) +#define CONDG COND(0xa) +#define CONDGE COND(0xb) +#define CONDGU COND(0xc) +#define CONDCC COND(0xd) +#define CONDPOS COND(0xe) +#define CONDVS COND(0xf) + +#define CONDGEU CONDCC +#define CONDLU CONDCS + +#define WDISP22(X) (((X) >> 2) & 0x3fffff) +#define WDISP19(X) (((X) >> 2) & 0x7ffff) + +/* The 10-bit branch displacement for CBCOND is split into two fields */ +static u32 WDISP10(u32 off) +{ + u32 ret = ((off >> 2) & 0xff) << 5; + + ret |= ((off >> (2 + 8)) & 0x03) << 19; + + return ret; +} + +#define CBCONDE CBCOND(0x09) +#define CBCONDLE CBCOND(0x0a) +#define CBCONDL CBCOND(0x0b) +#define CBCONDLEU CBCOND(0x0c) +#define CBCONDCS CBCOND(0x0d) +#define CBCONDN CBCOND(0x0e) +#define CBCONDVS CBCOND(0x0f) +#define CBCONDNE CBCOND(0x19) +#define CBCONDG CBCOND(0x1a) +#define CBCONDGE CBCOND(0x1b) +#define CBCONDGU CBCOND(0x1c) +#define CBCONDCC CBCOND(0x1d) +#define CBCONDPOS CBCOND(0x1e) +#define CBCONDVC CBCOND(0x1f) + +#define CBCONDGEU CBCONDCC +#define CBCONDLU CBCONDCS + +#define ANNUL (1 << 29) +#define XCC (1 << 21) + +#define BRANCH (F2(0, 1) | XCC) +#define CBCOND_OP (F2(0, 3) | XCC) + +#define BA (BRANCH | CONDA) +#define BG (BRANCH | CONDG) +#define BL (BRANCH | CONDL) +#define BLE (BRANCH | CONDLE) +#define BGU (BRANCH | CONDGU) +#define BLEU (BRANCH | CONDLEU) +#define BGE (BRANCH | CONDGE) +#define BGEU (BRANCH | CONDGEU) +#define BLU (BRANCH | CONDLU) +#define BE (BRANCH | CONDE) +#define BNE (BRANCH | CONDNE) + +#define SETHI(K, REG) \ + (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff)) +#define OR_LO(K, REG) \ + (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG)) + +#define ADD F3(2, 0x00) +#define AND F3(2, 0x01) +#define ANDCC F3(2, 0x11) +#define OR F3(2, 0x02) +#define XOR F3(2, 0x03) +#define SUB F3(2, 0x04) +#define SUBCC F3(2, 0x14) +#define MUL F3(2, 0x0a) +#define MULX F3(2, 0x09) +#define UDIVX F3(2, 0x0d) +#define DIV F3(2, 0x0e) +#define SLL F3(2, 0x25) +#define SLLX (F3(2, 0x25)|(1<<12)) +#define SRA F3(2, 0x27) +#define SRAX (F3(2, 0x27)|(1<<12)) +#define SRL F3(2, 0x26) +#define SRLX (F3(2, 0x26)|(1<<12)) +#define JMPL F3(2, 0x38) +#define SAVE F3(2, 0x3c) +#define RESTORE F3(2, 0x3d) +#define CALL F1(1) +#define BR F2(0, 0x01) +#define RD_Y F3(2, 0x28) +#define WR_Y F3(2, 0x30) + +#define LD32 F3(3, 0x00) +#define LD8 F3(3, 0x01) +#define LD16 F3(3, 0x02) +#define LD64 F3(3, 0x0b) +#define LD64A F3(3, 0x1b) +#define ST8 F3(3, 0x05) +#define ST16 F3(3, 0x06) +#define ST32 F3(3, 0x04) +#define ST64 F3(3, 0x0e) + +#define CAS F3(3, 0x3c) +#define CASX F3(3, 0x3e) + +#define LDPTR LD64 +#define BASE_STACKFRAME 176 + +#define LD32I (LD32 | IMMED) +#define LD8I (LD8 | IMMED) +#define LD16I (LD16 | IMMED) +#define LD64I (LD64 | IMMED) +#define LDPTRI (LDPTR | IMMED) +#define ST32I (ST32 | IMMED) + +struct jit_ctx { + struct bpf_prog *prog; + unsigned int *offset; + int idx; + int epilogue_offset; + bool tmp_1_used; + bool tmp_2_used; + bool tmp_3_used; + bool saw_frame_pointer; + bool saw_call; + bool saw_tail_call; + u32 *image; +}; + +#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) +#define TMP_REG_2 (MAX_BPF_JIT_REG + 1) +#define TMP_REG_3 (MAX_BPF_JIT_REG + 2) + +/* Map BPF registers to SPARC registers */ +static const int bpf2sparc[] = { + /* return value from in-kernel function, and exit value from eBPF */ + [BPF_REG_0] = O5, + + /* arguments from eBPF program to in-kernel function */ + [BPF_REG_1] = O0, + [BPF_REG_2] = O1, + [BPF_REG_3] = O2, + [BPF_REG_4] = O3, + [BPF_REG_5] = O4, + + /* callee saved registers that in-kernel function will preserve */ + [BPF_REG_6] = L0, + [BPF_REG_7] = L1, + [BPF_REG_8] = L2, + [BPF_REG_9] = L3, + + /* read-only frame pointer to access stack */ + [BPF_REG_FP] = L6, + + [BPF_REG_AX] = G7, + + /* temporary register for BPF JIT */ + [TMP_REG_1] = G1, + [TMP_REG_2] = G2, + [TMP_REG_3] = G3, +}; + +static void emit(const u32 insn, struct jit_ctx *ctx) +{ + if (ctx->image != NULL) + ctx->image[ctx->idx] = insn; + + ctx->idx++; +} + +static void emit_call(u32 *func, struct jit_ctx *ctx) +{ + if (ctx->image != NULL) { + void *here = &ctx->image[ctx->idx]; + unsigned int off; + + off = (void *)func - here; + ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff); + } + ctx->idx++; +} + +static void emit_nop(struct jit_ctx *ctx) +{ + emit(SETHI(0, G0), ctx); +} + +static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx) +{ + emit(OR | RS1(G0) | RS2(from) | RD(to), ctx); +} + +/* Emit 32-bit constant, zero extended. */ +static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx) +{ + emit(SETHI(K, reg), ctx); + emit(OR_LO(K, reg), ctx); +} + +/* Emit 32-bit constant, sign extended. */ +static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx) +{ + if (K >= 0) { + emit(SETHI(K, reg), ctx); + emit(OR_LO(K, reg), ctx); + } else { + u32 hbits = ~(u32) K; + u32 lbits = -0x400 | (u32) K; + + emit(SETHI(hbits, reg), ctx); + emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx); + } +} + +static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx) +{ + emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx); +} + +static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx) +{ + emit(opcode | RS1(a) | RS2(b) | RD(c), ctx); +} + +static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm, + struct jit_ctx *ctx) +{ + bool small_immed = is_simm13(imm); + unsigned int insn = opcode; + + insn |= RS1(dst) | RD(dst); + if (small_immed) { + emit(insn | IMMED | S13(imm), ctx); + } else { + unsigned int tmp = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_set_const_sext(imm, tmp, ctx); + emit(insn | RS2(tmp), ctx); + } +} + +static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm, + unsigned int dst, struct jit_ctx *ctx) +{ + bool small_immed = is_simm13(imm); + unsigned int insn = opcode; + + insn |= RS1(src) | RD(dst); + if (small_immed) { + emit(insn | IMMED | S13(imm), ctx); + } else { + unsigned int tmp = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_set_const_sext(imm, tmp, ctx); + emit(insn | RS2(tmp), ctx); + } +} + +static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx) +{ + if (K >= 0 && is_simm13(K)) { + /* or %g0, K, DEST */ + emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); + } else { + emit_set_const(K, dest, ctx); + } +} + +static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx) +{ + if (is_simm13(K)) { + /* or %g0, K, DEST */ + emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); + } else { + emit_set_const(K, dest, ctx); + } +} + +static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx) +{ + if (is_simm13(K)) { + /* or %g0, K, DEST */ + emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); + } else { + emit_set_const_sext(K, dest, ctx); + } +} + +static void analyze_64bit_constant(u32 high_bits, u32 low_bits, + int *hbsp, int *lbsp, int *abbasp) +{ + int lowest_bit_set, highest_bit_set, all_bits_between_are_set; + int i; + + lowest_bit_set = highest_bit_set = -1; + i = 0; + do { + if ((lowest_bit_set == -1) && ((low_bits >> i) & 1)) + lowest_bit_set = i; + if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1)) + highest_bit_set = (64 - i - 1); + } while (++i < 32 && (highest_bit_set == -1 || + lowest_bit_set == -1)); + if (i == 32) { + i = 0; + do { + if (lowest_bit_set == -1 && ((high_bits >> i) & 1)) + lowest_bit_set = i + 32; + if (highest_bit_set == -1 && + ((low_bits >> (32 - i - 1)) & 1)) + highest_bit_set = 32 - i - 1; + } while (++i < 32 && (highest_bit_set == -1 || + lowest_bit_set == -1)); + } + + all_bits_between_are_set = 1; + for (i = lowest_bit_set; i <= highest_bit_set; i++) { + if (i < 32) { + if ((low_bits & (1 << i)) != 0) + continue; + } else { + if ((high_bits & (1 << (i - 32))) != 0) + continue; + } + all_bits_between_are_set = 0; + break; + } + *hbsp = highest_bit_set; + *lbsp = lowest_bit_set; + *abbasp = all_bits_between_are_set; +} + +static unsigned long create_simple_focus_bits(unsigned long high_bits, + unsigned long low_bits, + int lowest_bit_set, int shift) +{ + long hi, lo; + + if (lowest_bit_set < 32) { + lo = (low_bits >> lowest_bit_set) << shift; + hi = ((high_bits << (32 - lowest_bit_set)) << shift); + } else { + lo = 0; + hi = ((high_bits >> (lowest_bit_set - 32)) << shift); + } + return hi | lo; +} + +static bool const64_is_2insns(unsigned long high_bits, + unsigned long low_bits) +{ + int highest_bit_set, lowest_bit_set, all_bits_between_are_set; + + if (high_bits == 0 || high_bits == 0xffffffff) + return true; + + analyze_64bit_constant(high_bits, low_bits, + &highest_bit_set, &lowest_bit_set, + &all_bits_between_are_set); + + if ((highest_bit_set == 63 || lowest_bit_set == 0) && + all_bits_between_are_set != 0) + return true; + + if (highest_bit_set - lowest_bit_set < 21) + return true; + + return false; +} + +static void sparc_emit_set_const64_quick2(unsigned long high_bits, + unsigned long low_imm, + unsigned int dest, + int shift_count, struct jit_ctx *ctx) +{ + emit_loadimm32(high_bits, dest, ctx); + + /* Now shift it up into place. */ + emit_alu_K(SLLX, dest, shift_count, ctx); + + /* If there is a low immediate part piece, finish up by + * putting that in as well. + */ + if (low_imm != 0) + emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx); +} + +static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx) +{ + int all_bits_between_are_set, lowest_bit_set, highest_bit_set; + unsigned int tmp = bpf2sparc[TMP_REG_1]; + u32 low_bits = (K & 0xffffffff); + u32 high_bits = (K >> 32); + + /* These two tests also take care of all of the one + * instruction cases. + */ + if (high_bits == 0xffffffff && (low_bits & 0x80000000)) + return emit_loadimm_sext(K, dest, ctx); + if (high_bits == 0x00000000) + return emit_loadimm32(K, dest, ctx); + + analyze_64bit_constant(high_bits, low_bits, &highest_bit_set, + &lowest_bit_set, &all_bits_between_are_set); + + /* 1) mov -1, %reg + * sllx %reg, shift, %reg + * 2) mov -1, %reg + * srlx %reg, shift, %reg + * 3) mov some_small_const, %reg + * sllx %reg, shift, %reg + */ + if (((highest_bit_set == 63 || lowest_bit_set == 0) && + all_bits_between_are_set != 0) || + ((highest_bit_set - lowest_bit_set) < 12)) { + int shift = lowest_bit_set; + long the_const = -1; + + if ((highest_bit_set != 63 && lowest_bit_set != 0) || + all_bits_between_are_set == 0) { + the_const = + create_simple_focus_bits(high_bits, low_bits, + lowest_bit_set, 0); + } else if (lowest_bit_set == 0) + shift = -(63 - highest_bit_set); + + emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx); + if (shift > 0) + emit_alu_K(SLLX, dest, shift, ctx); + else if (shift < 0) + emit_alu_K(SRLX, dest, -shift, ctx); + + return; + } + + /* Now a range of 22 or less bits set somewhere. + * 1) sethi %hi(focus_bits), %reg + * sllx %reg, shift, %reg + * 2) sethi %hi(focus_bits), %reg + * srlx %reg, shift, %reg + */ + if ((highest_bit_set - lowest_bit_set) < 21) { + unsigned long focus_bits = + create_simple_focus_bits(high_bits, low_bits, + lowest_bit_set, 10); + + emit(SETHI(focus_bits, dest), ctx); + + /* If lowest_bit_set == 10 then a sethi alone could + * have done it. + */ + if (lowest_bit_set < 10) + emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx); + else if (lowest_bit_set > 10) + emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx); + return; + } + + /* Ok, now 3 instruction sequences. */ + if (low_bits == 0) { + emit_loadimm32(high_bits, dest, ctx); + emit_alu_K(SLLX, dest, 32, ctx); + return; + } + + /* We may be able to do something quick + * when the constant is negated, so try that. + */ + if (const64_is_2insns((~high_bits) & 0xffffffff, + (~low_bits) & 0xfffffc00)) { + /* NOTE: The trailing bits get XOR'd so we need the + * non-negated bits, not the negated ones. + */ + unsigned long trailing_bits = low_bits & 0x3ff; + + if ((((~high_bits) & 0xffffffff) == 0 && + ((~low_bits) & 0x80000000) == 0) || + (((~high_bits) & 0xffffffff) == 0xffffffff && + ((~low_bits) & 0x80000000) != 0)) { + unsigned long fast_int = (~low_bits & 0xffffffff); + + if ((is_sethi(fast_int) && + (~high_bits & 0xffffffff) == 0)) { + emit(SETHI(fast_int, dest), ctx); + } else if (is_simm13(fast_int)) { + emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx); + } else { + emit_loadimm64(fast_int, dest, ctx); + } + } else { + u64 n = ((~low_bits) & 0xfffffc00) | + (((unsigned long)((~high_bits) & 0xffffffff))<<32); + emit_loadimm64(n, dest, ctx); + } + + low_bits = -0x400 | trailing_bits; + + emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx); + return; + } + + /* 1) sethi %hi(xxx), %reg + * or %reg, %lo(xxx), %reg + * sllx %reg, yyy, %reg + */ + if ((highest_bit_set - lowest_bit_set) < 32) { + unsigned long focus_bits = + create_simple_focus_bits(high_bits, low_bits, + lowest_bit_set, 0); + + /* So what we know is that the set bits straddle the + * middle of the 64-bit word. + */ + sparc_emit_set_const64_quick2(focus_bits, 0, dest, + lowest_bit_set, ctx); + return; + } + + /* 1) sethi %hi(high_bits), %reg + * or %reg, %lo(high_bits), %reg + * sllx %reg, 32, %reg + * or %reg, low_bits, %reg + */ + if (is_simm13(low_bits) && ((int)low_bits > 0)) { + sparc_emit_set_const64_quick2(high_bits, low_bits, + dest, 32, ctx); + return; + } + + /* Oh well, we tried... Do a full 64-bit decomposition. */ + ctx->tmp_1_used = true; + + emit_loadimm32(high_bits, tmp, ctx); + emit_loadimm32(low_bits, dest, ctx); + emit_alu_K(SLLX, tmp, 32, ctx); + emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx); +} + +static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx, + struct jit_ctx *ctx) +{ + unsigned int off = to_idx - from_idx; + + if (br_opc & XCC) + emit(br_opc | WDISP19(off << 2), ctx); + else + emit(br_opc | WDISP22(off << 2), ctx); +} + +static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx, + const u8 dst, const u8 src, struct jit_ctx *ctx) +{ + unsigned int off = to_idx - from_idx; + + emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx); +} + +static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx, + const u8 dst, s32 imm, struct jit_ctx *ctx) +{ + unsigned int off = to_idx - from_idx; + + emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx); +} + +#define emit_read_y(REG, CTX) emit(RD_Y | RD(REG), CTX) +#define emit_write_y(REG, CTX) emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX) + +#define emit_cmp(R1, R2, CTX) \ + emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX) + +#define emit_cmpi(R1, IMM, CTX) \ + emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX) + +#define emit_btst(R1, R2, CTX) \ + emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX) + +#define emit_btsti(R1, IMM, CTX) \ + emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX) + +static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src, + const s32 imm, bool is_imm, int branch_dst, + struct jit_ctx *ctx) +{ + bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0; + const u8 tmp = bpf2sparc[TMP_REG_1]; + + branch_dst = ctx->offset[branch_dst]; + + if (!is_simm10(branch_dst - ctx->idx) || + BPF_OP(code) == BPF_JSET) + use_cbcond = false; + + if (is_imm) { + bool fits = true; + + if (use_cbcond) { + if (!is_simm5(imm)) + fits = false; + } else if (!is_simm13(imm)) { + fits = false; + } + if (!fits) { + ctx->tmp_1_used = true; + emit_loadimm_sext(imm, tmp, ctx); + src = tmp; + is_imm = false; + } + } + + if (!use_cbcond) { + u32 br_opcode; + + if (BPF_OP(code) == BPF_JSET) { + if (is_imm) + emit_btsti(dst, imm, ctx); + else + emit_btst(dst, src, ctx); + } else { + if (is_imm) + emit_cmpi(dst, imm, ctx); + else + emit_cmp(dst, src, ctx); + } + switch (BPF_OP(code)) { + case BPF_JEQ: + br_opcode = BE; + break; + case BPF_JGT: + br_opcode = BGU; + break; + case BPF_JLT: + br_opcode = BLU; + break; + case BPF_JGE: + br_opcode = BGEU; + break; + case BPF_JLE: + br_opcode = BLEU; + break; + case BPF_JSET: + case BPF_JNE: + br_opcode = BNE; + break; + case BPF_JSGT: + br_opcode = BG; + break; + case BPF_JSLT: + br_opcode = BL; + break; + case BPF_JSGE: + br_opcode = BGE; + break; + case BPF_JSLE: + br_opcode = BLE; + break; + default: + /* Make sure we dont leak kernel information to the + * user. + */ + return -EFAULT; + } + emit_branch(br_opcode, ctx->idx, branch_dst, ctx); + emit_nop(ctx); + } else { + u32 cbcond_opcode; + + switch (BPF_OP(code)) { + case BPF_JEQ: + cbcond_opcode = CBCONDE; + break; + case BPF_JGT: + cbcond_opcode = CBCONDGU; + break; + case BPF_JLT: + cbcond_opcode = CBCONDLU; + break; + case BPF_JGE: + cbcond_opcode = CBCONDGEU; + break; + case BPF_JLE: + cbcond_opcode = CBCONDLEU; + break; + case BPF_JNE: + cbcond_opcode = CBCONDNE; + break; + case BPF_JSGT: + cbcond_opcode = CBCONDG; + break; + case BPF_JSLT: + cbcond_opcode = CBCONDL; + break; + case BPF_JSGE: + cbcond_opcode = CBCONDGE; + break; + case BPF_JSLE: + cbcond_opcode = CBCONDLE; + break; + default: + /* Make sure we dont leak kernel information to the + * user. + */ + return -EFAULT; + } + cbcond_opcode |= CBCOND_OP; + if (is_imm) + emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst, + dst, imm, ctx); + else + emit_cbcond(cbcond_opcode, ctx->idx, branch_dst, + dst, src, ctx); + } + return 0; +} + +/* Just skip the save instruction and the ctx register move. */ +#define BPF_TAILCALL_PROLOGUE_SKIP 32 +#define BPF_TAILCALL_CNT_SP_OFF (STACK_BIAS + 128) + +static void build_prologue(struct jit_ctx *ctx) +{ + s32 stack_needed = BASE_STACKFRAME; + + if (ctx->saw_frame_pointer || ctx->saw_tail_call) { + struct bpf_prog *prog = ctx->prog; + u32 stack_depth; + + stack_depth = prog->aux->stack_depth; + stack_needed += round_up(stack_depth, 16); + } + + if (ctx->saw_tail_call) + stack_needed += 8; + + /* save %sp, -176, %sp */ + emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx); + + /* tail_call_cnt = 0 */ + if (ctx->saw_tail_call) { + u32 off = BPF_TAILCALL_CNT_SP_OFF; + + emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx); + } else { + emit_nop(ctx); + } + if (ctx->saw_frame_pointer) { + const u8 vfp = bpf2sparc[BPF_REG_FP]; + + emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx); + } else { + emit_nop(ctx); + } + + emit_reg_move(I0, O0, ctx); + emit_reg_move(I1, O1, ctx); + emit_reg_move(I2, O2, ctx); + emit_reg_move(I3, O3, ctx); + emit_reg_move(I4, O4, ctx); + /* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */ +} + +static void build_epilogue(struct jit_ctx *ctx) +{ + ctx->epilogue_offset = ctx->idx; + + /* ret (jmpl %i7 + 8, %g0) */ + emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx); + + /* restore %i5, %g0, %o0 */ + emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx); +} + +static void emit_tail_call(struct jit_ctx *ctx) +{ + const u8 bpf_array = bpf2sparc[BPF_REG_2]; + const u8 bpf_index = bpf2sparc[BPF_REG_3]; + const u8 tmp = bpf2sparc[TMP_REG_1]; + u32 off; + + ctx->saw_tail_call = true; + + off = offsetof(struct bpf_array, map.max_entries); + emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx); + emit_cmp(bpf_index, tmp, ctx); +#define OFFSET1 17 + emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx); + emit_nop(ctx); + + off = BPF_TAILCALL_CNT_SP_OFF; + emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx); + emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx); +#define OFFSET2 13 + emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET2, ctx); + emit_nop(ctx); + + emit_alu_K(ADD, tmp, 1, ctx); + off = BPF_TAILCALL_CNT_SP_OFF; + emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx); + + emit_alu3_K(SLL, bpf_index, 3, tmp, ctx); + emit_alu(ADD, bpf_array, tmp, ctx); + off = offsetof(struct bpf_array, ptrs); + emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx); + + emit_cmpi(tmp, 0, ctx); +#define OFFSET3 5 + emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx); + emit_nop(ctx); + + off = offsetof(struct bpf_prog, bpf_func); + emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx); + + off = BPF_TAILCALL_PROLOGUE_SKIP; + emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx); + emit_nop(ctx); +} + +static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) +{ + const u8 code = insn->code; + const u8 dst = bpf2sparc[insn->dst_reg]; + const u8 src = bpf2sparc[insn->src_reg]; + const int i = insn - ctx->prog->insnsi; + const s16 off = insn->off; + const s32 imm = insn->imm; + + if (insn->src_reg == BPF_REG_FP) + ctx->saw_frame_pointer = true; + + switch (code) { + /* dst = src */ + case BPF_ALU | BPF_MOV | BPF_X: + emit_alu3_K(SRL, src, 0, dst, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_ALU64 | BPF_MOV | BPF_X: + emit_reg_move(src, dst, ctx); + break; + /* dst = dst OP src */ + case BPF_ALU | BPF_ADD | BPF_X: + case BPF_ALU64 | BPF_ADD | BPF_X: + emit_alu(ADD, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_SUB | BPF_X: + case BPF_ALU64 | BPF_SUB | BPF_X: + emit_alu(SUB, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_AND | BPF_X: + case BPF_ALU64 | BPF_AND | BPF_X: + emit_alu(AND, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_OR | BPF_X: + case BPF_ALU64 | BPF_OR | BPF_X: + emit_alu(OR, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_XOR | BPF_X: + case BPF_ALU64 | BPF_XOR | BPF_X: + emit_alu(XOR, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_MUL | BPF_X: + emit_alu(MUL, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_MUL | BPF_X: + emit_alu(MULX, src, dst, ctx); + break; + case BPF_ALU | BPF_DIV | BPF_X: + emit_write_y(G0, ctx); + emit_alu(DIV, src, dst, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_ALU64 | BPF_DIV | BPF_X: + emit_alu(UDIVX, src, dst, ctx); + break; + case BPF_ALU | BPF_MOD | BPF_X: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_write_y(G0, ctx); + emit_alu3(DIV, dst, src, tmp, ctx); + emit_alu3(MULX, tmp, src, tmp, ctx); + emit_alu3(SUB, dst, tmp, dst, ctx); + goto do_alu32_trunc; + } + case BPF_ALU64 | BPF_MOD | BPF_X: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_alu3(UDIVX, dst, src, tmp, ctx); + emit_alu3(MULX, tmp, src, tmp, ctx); + emit_alu3(SUB, dst, tmp, dst, ctx); + break; + } + case BPF_ALU | BPF_LSH | BPF_X: + emit_alu(SLL, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_LSH | BPF_X: + emit_alu(SLLX, src, dst, ctx); + break; + case BPF_ALU | BPF_RSH | BPF_X: + emit_alu(SRL, src, dst, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_ALU64 | BPF_RSH | BPF_X: + emit_alu(SRLX, src, dst, ctx); + break; + case BPF_ALU | BPF_ARSH | BPF_X: + emit_alu(SRA, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_ARSH | BPF_X: + emit_alu(SRAX, src, dst, ctx); + break; + + /* dst = -dst */ + case BPF_ALU | BPF_NEG: + case BPF_ALU64 | BPF_NEG: + emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx); + goto do_alu32_trunc; + + case BPF_ALU | BPF_END | BPF_FROM_BE: + switch (imm) { + case 16: + emit_alu_K(SLL, dst, 16, ctx); + emit_alu_K(SRL, dst, 16, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case 32: + if (!ctx->prog->aux->verifier_zext) + emit_alu_K(SRL, dst, 0, ctx); + break; + case 64: + /* nop */ + break; + + } + break; + + /* dst = BSWAP##imm(dst) */ + case BPF_ALU | BPF_END | BPF_FROM_LE: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + const u8 tmp2 = bpf2sparc[TMP_REG_2]; + + ctx->tmp_1_used = true; + switch (imm) { + case 16: + emit_alu3_K(AND, dst, 0xff, tmp, ctx); + emit_alu3_K(SRL, dst, 8, dst, ctx); + emit_alu3_K(AND, dst, 0xff, dst, ctx); + emit_alu3_K(SLL, tmp, 8, tmp, ctx); + emit_alu(OR, tmp, dst, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + + case 32: + ctx->tmp_2_used = true; + emit_alu3_K(SRL, dst, 24, tmp, ctx); /* tmp = dst >> 24 */ + emit_alu3_K(SRL, dst, 16, tmp2, ctx); /* tmp2 = dst >> 16 */ + emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */ + emit_alu3_K(SLL, tmp2, 8, tmp2, ctx); /* tmp2 = tmp2 << 8 */ + emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */ + emit_alu3_K(SRL, dst, 8, tmp2, ctx); /* tmp2 = dst >> 8 */ + emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */ + emit_alu3_K(SLL, tmp2, 16, tmp2, ctx); /* tmp2 = tmp2 << 16 */ + emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */ + emit_alu3_K(AND, dst, 0xff, dst, ctx); /* dst = dst & 0xff */ + emit_alu3_K(SLL, dst, 24, dst, ctx); /* dst = dst << 24 */ + emit_alu(OR, tmp, dst, ctx); /* dst = dst | tmp */ + if (insn_is_zext(&insn[1])) + return 1; + break; + + case 64: + emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx); + emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx); + emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx); + break; + } + break; + } + /* dst = imm */ + case BPF_ALU | BPF_MOV | BPF_K: + emit_loadimm32(imm, dst, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_ALU64 | BPF_MOV | BPF_K: + emit_loadimm_sext(imm, dst, ctx); + break; + /* dst = dst OP imm */ + case BPF_ALU | BPF_ADD | BPF_K: + case BPF_ALU64 | BPF_ADD | BPF_K: + emit_alu_K(ADD, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_SUB | BPF_K: + case BPF_ALU64 | BPF_SUB | BPF_K: + emit_alu_K(SUB, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_AND | BPF_K: + case BPF_ALU64 | BPF_AND | BPF_K: + emit_alu_K(AND, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_OR | BPF_K: + case BPF_ALU64 | BPF_OR | BPF_K: + emit_alu_K(OR, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_XOR | BPF_K: + case BPF_ALU64 | BPF_XOR | BPF_K: + emit_alu_K(XOR, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_MUL | BPF_K: + emit_alu_K(MUL, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_MUL | BPF_K: + emit_alu_K(MULX, dst, imm, ctx); + break; + case BPF_ALU | BPF_DIV | BPF_K: + if (imm == 0) + return -EINVAL; + + emit_write_y(G0, ctx); + emit_alu_K(DIV, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_DIV | BPF_K: + if (imm == 0) + return -EINVAL; + + emit_alu_K(UDIVX, dst, imm, ctx); + break; + case BPF_ALU64 | BPF_MOD | BPF_K: + case BPF_ALU | BPF_MOD | BPF_K: { + const u8 tmp = bpf2sparc[TMP_REG_2]; + unsigned int div; + + if (imm == 0) + return -EINVAL; + + div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV; + + ctx->tmp_2_used = true; + + if (BPF_CLASS(code) != BPF_ALU64) + emit_write_y(G0, ctx); + if (is_simm13(imm)) { + emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx); + emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx); + emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx); + } else { + const u8 tmp1 = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_set_const_sext(imm, tmp1, ctx); + emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx); + emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx); + emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx); + } + goto do_alu32_trunc; + } + case BPF_ALU | BPF_LSH | BPF_K: + emit_alu_K(SLL, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_LSH | BPF_K: + emit_alu_K(SLLX, dst, imm, ctx); + break; + case BPF_ALU | BPF_RSH | BPF_K: + emit_alu_K(SRL, dst, imm, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_ALU64 | BPF_RSH | BPF_K: + emit_alu_K(SRLX, dst, imm, ctx); + break; + case BPF_ALU | BPF_ARSH | BPF_K: + emit_alu_K(SRA, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_ARSH | BPF_K: + emit_alu_K(SRAX, dst, imm, ctx); + break; + + do_alu32_trunc: + if (BPF_CLASS(code) == BPF_ALU && + !ctx->prog->aux->verifier_zext) + emit_alu_K(SRL, dst, 0, ctx); + break; + + /* JUMP off */ + case BPF_JMP | BPF_JA: + emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx); + emit_nop(ctx); + break; + /* IF (dst COND src) JUMP off */ + case BPF_JMP | BPF_JEQ | BPF_X: + case BPF_JMP | BPF_JGT | BPF_X: + case BPF_JMP | BPF_JLT | BPF_X: + case BPF_JMP | BPF_JGE | BPF_X: + case BPF_JMP | BPF_JLE | BPF_X: + case BPF_JMP | BPF_JNE | BPF_X: + case BPF_JMP | BPF_JSGT | BPF_X: + case BPF_JMP | BPF_JSLT | BPF_X: + case BPF_JMP | BPF_JSGE | BPF_X: + case BPF_JMP | BPF_JSLE | BPF_X: + case BPF_JMP | BPF_JSET | BPF_X: { + int err; + + err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx); + if (err) + return err; + break; + } + /* IF (dst COND imm) JUMP off */ + case BPF_JMP | BPF_JEQ | BPF_K: + case BPF_JMP | BPF_JGT | BPF_K: + case BPF_JMP | BPF_JLT | BPF_K: + case BPF_JMP | BPF_JGE | BPF_K: + case BPF_JMP | BPF_JLE | BPF_K: + case BPF_JMP | BPF_JNE | BPF_K: + case BPF_JMP | BPF_JSGT | BPF_K: + case BPF_JMP | BPF_JSLT | BPF_K: + case BPF_JMP | BPF_JSGE | BPF_K: + case BPF_JMP | BPF_JSLE | BPF_K: + case BPF_JMP | BPF_JSET | BPF_K: { + int err; + + err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx); + if (err) + return err; + break; + } + + /* function call */ + case BPF_JMP | BPF_CALL: + { + u8 *func = ((u8 *)__bpf_call_base) + imm; + + ctx->saw_call = true; + + emit_call((u32 *)func, ctx); + emit_nop(ctx); + + emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx); + break; + } + + /* tail call */ + case BPF_JMP | BPF_TAIL_CALL: + emit_tail_call(ctx); + break; + + /* function return */ + case BPF_JMP | BPF_EXIT: + /* Optimization: when last instruction is EXIT, + simply fallthrough to epilogue. */ + if (i == ctx->prog->len - 1) + break; + emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx); + emit_nop(ctx); + break; + + /* dst = imm64 */ + case BPF_LD | BPF_IMM | BPF_DW: + { + const struct bpf_insn insn1 = insn[1]; + u64 imm64; + + imm64 = (u64)insn1.imm << 32 | (u32)imm; + emit_loadimm64(imm64, dst, ctx); + + return 1; + } + + /* LDX: dst = *(size *)(src + off) */ + case BPF_LDX | BPF_MEM | BPF_W: + case BPF_LDX | BPF_MEM | BPF_H: + case BPF_LDX | BPF_MEM | BPF_B: + case BPF_LDX | BPF_MEM | BPF_DW: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + u32 opcode = 0, rs2; + + ctx->tmp_1_used = true; + switch (BPF_SIZE(code)) { + case BPF_W: + opcode = LD32; + break; + case BPF_H: + opcode = LD16; + break; + case BPF_B: + opcode = LD8; + break; + case BPF_DW: + opcode = LD64; + break; + } + + if (is_simm13(off)) { + opcode |= IMMED; + rs2 = S13(off); + } else { + emit_loadimm(off, tmp, ctx); + rs2 = RS2(tmp); + } + emit(opcode | RS1(src) | rs2 | RD(dst), ctx); + if (opcode != LD64 && insn_is_zext(&insn[1])) + return 1; + break; + } + /* speculation barrier */ + case BPF_ST | BPF_NOSPEC: + break; + /* ST: *(size *)(dst + off) = imm */ + case BPF_ST | BPF_MEM | BPF_W: + case BPF_ST | BPF_MEM | BPF_H: + case BPF_ST | BPF_MEM | BPF_B: + case BPF_ST | BPF_MEM | BPF_DW: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + const u8 tmp2 = bpf2sparc[TMP_REG_2]; + u32 opcode = 0, rs2; + + if (insn->dst_reg == BPF_REG_FP) + ctx->saw_frame_pointer = true; + + ctx->tmp_2_used = true; + emit_loadimm(imm, tmp2, ctx); + + switch (BPF_SIZE(code)) { + case BPF_W: + opcode = ST32; + break; + case BPF_H: + opcode = ST16; + break; + case BPF_B: + opcode = ST8; + break; + case BPF_DW: + opcode = ST64; + break; + } + + if (is_simm13(off)) { + opcode |= IMMED; + rs2 = S13(off); + } else { + ctx->tmp_1_used = true; + emit_loadimm(off, tmp, ctx); + rs2 = RS2(tmp); + } + emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx); + break; + } + + /* STX: *(size *)(dst + off) = src */ + case BPF_STX | BPF_MEM | BPF_W: + case BPF_STX | BPF_MEM | BPF_H: + case BPF_STX | BPF_MEM | BPF_B: + case BPF_STX | BPF_MEM | BPF_DW: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + u32 opcode = 0, rs2; + + if (insn->dst_reg == BPF_REG_FP) + ctx->saw_frame_pointer = true; + + switch (BPF_SIZE(code)) { + case BPF_W: + opcode = ST32; + break; + case BPF_H: + opcode = ST16; + break; + case BPF_B: + opcode = ST8; + break; + case BPF_DW: + opcode = ST64; + break; + } + if (is_simm13(off)) { + opcode |= IMMED; + rs2 = S13(off); + } else { + ctx->tmp_1_used = true; + emit_loadimm(off, tmp, ctx); + rs2 = RS2(tmp); + } + emit(opcode | RS1(dst) | rs2 | RD(src), ctx); + break; + } + + case BPF_STX | BPF_ATOMIC | BPF_W: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + const u8 tmp2 = bpf2sparc[TMP_REG_2]; + const u8 tmp3 = bpf2sparc[TMP_REG_3]; + + if (insn->imm != BPF_ADD) { + pr_err_once("unknown atomic op %02x\n", insn->imm); + return -EINVAL; + } + + /* lock *(u32 *)(dst + off) += src */ + + if (insn->dst_reg == BPF_REG_FP) + ctx->saw_frame_pointer = true; + + ctx->tmp_1_used = true; + ctx->tmp_2_used = true; + ctx->tmp_3_used = true; + emit_loadimm(off, tmp, ctx); + emit_alu3(ADD, dst, tmp, tmp, ctx); + + emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx); + emit_alu3(ADD, tmp2, src, tmp3, ctx); + emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx); + emit_cmp(tmp2, tmp3, ctx); + emit_branch(BNE, 4, 0, ctx); + emit_nop(ctx); + break; + } + /* STX XADD: lock *(u64 *)(dst + off) += src */ + case BPF_STX | BPF_ATOMIC | BPF_DW: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + const u8 tmp2 = bpf2sparc[TMP_REG_2]; + const u8 tmp3 = bpf2sparc[TMP_REG_3]; + + if (insn->imm != BPF_ADD) { + pr_err_once("unknown atomic op %02x\n", insn->imm); + return -EINVAL; + } + + if (insn->dst_reg == BPF_REG_FP) + ctx->saw_frame_pointer = true; + + ctx->tmp_1_used = true; + ctx->tmp_2_used = true; + ctx->tmp_3_used = true; + emit_loadimm(off, tmp, ctx); + emit_alu3(ADD, dst, tmp, tmp, ctx); + + emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx); + emit_alu3(ADD, tmp2, src, tmp3, ctx); + emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx); + emit_cmp(tmp2, tmp3, ctx); + emit_branch(BNE, 4, 0, ctx); + emit_nop(ctx); + break; + } + + default: + pr_err_once("unknown opcode %02x\n", code); + return -EINVAL; + } + + return 0; +} + +static int build_body(struct jit_ctx *ctx) +{ + const struct bpf_prog *prog = ctx->prog; + int i; + + for (i = 0; i < prog->len; i++) { + const struct bpf_insn *insn = &prog->insnsi[i]; + int ret; + + ret = build_insn(insn, ctx); + + if (ret > 0) { + i++; + ctx->offset[i] = ctx->idx; + continue; + } + ctx->offset[i] = ctx->idx; + if (ret) + return ret; + } + return 0; +} + +static void jit_fill_hole(void *area, unsigned int size) +{ + u32 *ptr; + /* We are guaranteed to have aligned memory. */ + for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) + *ptr++ = 0x91d02005; /* ta 5 */ +} + +bool bpf_jit_needs_zext(void) +{ + return true; +} + +struct sparc64_jit_data { + struct bpf_binary_header *header; + u8 *image; + struct jit_ctx ctx; +}; + +struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +{ + struct bpf_prog *tmp, *orig_prog = prog; + struct sparc64_jit_data *jit_data; + struct bpf_binary_header *header; + u32 prev_image_size, image_size; + bool tmp_blinded = false; + bool extra_pass = false; + struct jit_ctx ctx; + u8 *image_ptr; + int pass, i; + + if (!prog->jit_requested) + return orig_prog; + + tmp = bpf_jit_blind_constants(prog); + /* If blinding was requested and we failed during blinding, + * we must fall back to the interpreter. + */ + if (IS_ERR(tmp)) + return orig_prog; + if (tmp != prog) { + tmp_blinded = true; + prog = tmp; + } + + jit_data = prog->aux->jit_data; + if (!jit_data) { + jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); + if (!jit_data) { + prog = orig_prog; + goto out; + } + prog->aux->jit_data = jit_data; + } + if (jit_data->ctx.offset) { + ctx = jit_data->ctx; + image_ptr = jit_data->image; + header = jit_data->header; + extra_pass = true; + image_size = sizeof(u32) * ctx.idx; + prev_image_size = image_size; + pass = 1; + goto skip_init_ctx; + } + + memset(&ctx, 0, sizeof(ctx)); + ctx.prog = prog; + + ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL); + if (ctx.offset == NULL) { + prog = orig_prog; + goto out_off; + } + + /* Longest sequence emitted is for bswap32, 12 instructions. Pre-cook + * the offset array so that we converge faster. + */ + for (i = 0; i < prog->len; i++) + ctx.offset[i] = i * (12 * 4); + + prev_image_size = ~0U; + for (pass = 1; pass < 40; pass++) { + ctx.idx = 0; + + build_prologue(&ctx); + if (build_body(&ctx)) { + prog = orig_prog; + goto out_off; + } + build_epilogue(&ctx); + + if (bpf_jit_enable > 1) + pr_info("Pass %d: size = %u, seen = [%c%c%c%c%c%c]\n", pass, + ctx.idx * 4, + ctx.tmp_1_used ? '1' : ' ', + ctx.tmp_2_used ? '2' : ' ', + ctx.tmp_3_used ? '3' : ' ', + ctx.saw_frame_pointer ? 'F' : ' ', + ctx.saw_call ? 'C' : ' ', + ctx.saw_tail_call ? 'T' : ' '); + + if (ctx.idx * 4 == prev_image_size) + break; + prev_image_size = ctx.idx * 4; + cond_resched(); + } + + /* Now we know the actual image size. */ + image_size = sizeof(u32) * ctx.idx; + header = bpf_jit_binary_alloc(image_size, &image_ptr, + sizeof(u32), jit_fill_hole); + if (header == NULL) { + prog = orig_prog; + goto out_off; + } + + ctx.image = (u32 *)image_ptr; +skip_init_ctx: + ctx.idx = 0; + + build_prologue(&ctx); + + if (build_body(&ctx)) { + bpf_jit_binary_free(header); + prog = orig_prog; + goto out_off; + } + + build_epilogue(&ctx); + + if (ctx.idx * 4 != prev_image_size) { + pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n", + prev_image_size, ctx.idx * 4); + bpf_jit_binary_free(header); + prog = orig_prog; + goto out_off; + } + + if (bpf_jit_enable > 1) + bpf_jit_dump(prog->len, image_size, pass, ctx.image); + + bpf_flush_icache(header, (u8 *)header + header->size); + + if (!prog->is_func || extra_pass) { + if (bpf_jit_binary_lock_ro(header)) { + bpf_jit_binary_free(header); + prog = orig_prog; + goto out_off; + } + } else { + jit_data->ctx = ctx; + jit_data->image = image_ptr; + jit_data->header = header; + } + + prog->bpf_func = (void *)ctx.image; + prog->jited = 1; + prog->jited_len = image_size; + + if (!prog->is_func || extra_pass) { + bpf_prog_fill_jited_linfo(prog, ctx.offset); +out_off: + kfree(ctx.offset); + kfree(jit_data); + prog->aux->jit_data = NULL; + } +out: + if (tmp_blinded) + bpf_jit_prog_release_other(prog, prog == orig_prog ? + tmp : orig_prog); + return prog; +} diff --git a/arch/sparc/oprofile/Makefile b/arch/sparc/oprofile/Makefile deleted file mode 100644 index e9feca1ca28b..000000000000 --- a/arch/sparc/oprofile/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -obj-$(CONFIG_OPROFILE) += oprofile.o - -DRIVER_OBJS = $(addprefix ../../../drivers/oprofile/, \ - oprof.o cpu_buffer.o buffer_sync.o \ - event_buffer.o oprofile_files.o \ - oprofilefs.o oprofile_stats.o \ - timer_int.o ) - -oprofile-y := $(DRIVER_OBJS) init.o diff --git a/arch/sparc/oprofile/init.c b/arch/sparc/oprofile/init.c deleted file mode 100644 index f9024bccff16..000000000000 --- a/arch/sparc/oprofile/init.c +++ /dev/null @@ -1,87 +0,0 @@ -/** - * @file init.c - * - * @remark Copyright 2002 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon <levon@movementarian.org> - */ - -#include <linux/kernel.h> -#include <linux/oprofile.h> -#include <linux/errno.h> -#include <linux/init.h> -#include <linux/param.h> /* for HZ */ - -#ifdef CONFIG_SPARC64 -#include <linux/notifier.h> -#include <linux/rcupdate.h> -#include <linux/kdebug.h> -#include <asm/nmi.h> - -static int profile_timer_exceptions_notify(struct notifier_block *self, - unsigned long val, void *data) -{ - struct die_args *args = data; - int ret = NOTIFY_DONE; - - switch (val) { - case DIE_NMI: - oprofile_add_sample(args->regs, 0); - ret = NOTIFY_STOP; - break; - default: - break; - } - return ret; -} - -static struct notifier_block profile_timer_exceptions_nb = { - .notifier_call = profile_timer_exceptions_notify, -}; - -static int timer_start(void) -{ - if (register_die_notifier(&profile_timer_exceptions_nb)) - return 1; - nmi_adjust_hz(HZ); - return 0; -} - - -static void timer_stop(void) -{ - nmi_adjust_hz(1); - unregister_die_notifier(&profile_timer_exceptions_nb); - synchronize_sched(); /* Allow already-started NMIs to complete. */ -} - -static int op_nmi_timer_init(struct oprofile_operations *ops) -{ - if (atomic_read(&nmi_active) <= 0) - return -ENODEV; - - ops->start = timer_start; - ops->stop = timer_stop; - ops->cpu_type = "timer"; - printk(KERN_INFO "oprofile: Using perfctr NMI timer interrupt.\n"); - return 0; -} -#endif - -int __init oprofile_arch_init(struct oprofile_operations *ops) -{ - int ret = -ENODEV; - -#ifdef CONFIG_SPARC64 - ret = op_nmi_timer_init(ops); - if (!ret) - return ret; -#endif - - return ret; -} - -void oprofile_arch_exit(void) -{ -} diff --git a/arch/sparc/power/Makefile b/arch/sparc/power/Makefile index 3201ace0ddbd..d8f75e7cb05f 100644 --- a/arch/sparc/power/Makefile +++ b/arch/sparc/power/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only # Makefile for Sparc-specific hibernate files. obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate_asm.o diff --git a/arch/sparc/power/hibernate.c b/arch/sparc/power/hibernate.c index 42b0b8ce699a..da8e2bc2e516 100644 --- a/arch/sparc/power/hibernate.c +++ b/arch/sparc/power/hibernate.c @@ -1,19 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 /* * hibernate.c: Hibernaton support specific for sparc64. * * Copyright (C) 2013 Kirill V Tkhai (tkhai@yandex.ru) */ +#include <linux/suspend.h> #include <linux/mm.h> #include <asm/hibernate.h> #include <asm/visasm.h> #include <asm/page.h> +#include <asm/sections.h> #include <asm/tlb.h> -/* References to section boundaries */ -extern const void __nosave_begin, __nosave_end; - struct saved_context saved_context; /* @@ -37,6 +37,5 @@ void restore_processor_state(void) { struct mm_struct *mm = current->active_mm; - load_secondary_context(mm); - tsb_context_switch(mm); + tsb_context_switch_ctx(mm, CTX_HWBITS(mm->context)); } diff --git a/arch/sparc/power/hibernate_asm.S b/arch/sparc/power/hibernate_asm.S index 79942166df84..8cfaf5b6a32e 100644 --- a/arch/sparc/power/hibernate_asm.S +++ b/arch/sparc/power/hibernate_asm.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * hibernate_asm.S: Hibernaton support specific for sparc64. * @@ -54,8 +55,8 @@ ENTRY(swsusp_arch_resume) nop /* Write PAGE_OFFSET to %g7 */ - sethi %uhi(PAGE_OFFSET), %g7 - sllx %g7, 32, %g7 + sethi %hi(PAGE_OFFSET), %g7 + ldx [%g7 + %lo(PAGE_OFFSET)], %g7 setuw (PAGE_SIZE-8), %g3 diff --git a/arch/sparc/prom/Makefile b/arch/sparc/prom/Makefile index 020300b18c0b..92db8bb4ad4c 100644 --- a/arch/sparc/prom/Makefile +++ b/arch/sparc/prom/Makefile @@ -1,8 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 # Makefile for the Sun Boot PROM interface library under # Linux. # -asflags := -ansi -ccflags := -Werror lib-y := bootstr_$(BITS).o lib-y += init_$(BITS).o diff --git a/arch/sparc/prom/bootstr_32.c b/arch/sparc/prom/bootstr_32.c index d2b49d2365e7..1c7cd258b0dc 100644 --- a/arch/sparc/prom/bootstr_32.c +++ b/arch/sparc/prom/bootstr_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * bootstr.c: Boot string/argument acquisition from the PROM. * @@ -51,7 +52,7 @@ prom_getbootargs(void) * V3 PROM cannot supply as with more than 128 bytes * of an argument. But a smart bootstrap loader can. */ - strlcpy(barg_buf, *romvec->pv_v2bootargs.bootargs, sizeof(barg_buf)); + strscpy(barg_buf, *romvec->pv_v2bootargs.bootargs, sizeof(barg_buf)); break; default: break; diff --git a/arch/sparc/prom/bootstr_64.c b/arch/sparc/prom/bootstr_64.c index ab9ccc63b388..f1cc34d99eec 100644 --- a/arch/sparc/prom/bootstr_64.c +++ b/arch/sparc/prom/bootstr_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * bootstr.c: Boot string/argument acquisition from the PROM. * @@ -14,7 +15,10 @@ * the .bss section or it will break things. */ -#define BARG_LEN 256 +/* We limit BARG_LEN to 1024 because this is the size of the + * 'barg_out' command line buffer in the SILO bootloader. + */ +#define BARG_LEN 1024 struct { int bootstr_len; int bootstr_valid; diff --git a/arch/sparc/prom/cif.S b/arch/sparc/prom/cif.S index 9c86b4b7d429..dd06bb1fb90c 100644 --- a/arch/sparc/prom/cif.S +++ b/arch/sparc/prom/cif.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* cif.S: PROM entry/exit assembler trampolines. * * Copyright (C) 1996, 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) @@ -11,11 +12,10 @@ .text .globl prom_cif_direct prom_cif_direct: + save %sp, -192, %sp sethi %hi(p1275buf), %o1 or %o1, %lo(p1275buf), %o1 - ldx [%o1 + 0x0010], %o2 ! prom_cif_stack - save %o2, -192, %sp - ldx [%i1 + 0x0008], %l2 ! prom_cif_handler + ldx [%o1 + 0x0008], %l2 ! prom_cif_handler mov %g4, %l0 mov %g5, %l1 mov %g6, %l3 diff --git a/arch/sparc/prom/console_32.c b/arch/sparc/prom/console_32.c index 1cfb50f4cb9c..6404e5bbb7e6 100644 --- a/arch/sparc/prom/console_32.c +++ b/arch/sparc/prom/console_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * console.c: Routines that deal with sending and receiving IO * to/from the current console device using the PROM. diff --git a/arch/sparc/prom/console_64.c b/arch/sparc/prom/console_64.c index f95edcc54fd5..86dace787ae6 100644 --- a/arch/sparc/prom/console_64.c +++ b/arch/sparc/prom/console_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* console.c: Routines that deal with sending and receiving IO * to/from the current console device using the PROM. * diff --git a/arch/sparc/prom/init_32.c b/arch/sparc/prom/init_32.c index 9ac30c2b7dba..d20470166cb1 100644 --- a/arch/sparc/prom/init_32.c +++ b/arch/sparc/prom/init_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * init.c: Initialize internal variables used by the PROM * library functions. diff --git a/arch/sparc/prom/init_64.c b/arch/sparc/prom/init_64.c index d95db755828f..f7b8a1a865b8 100644 --- a/arch/sparc/prom/init_64.c +++ b/arch/sparc/prom/init_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * init.c: Initialize internal variables used by the PROM * library functions. @@ -25,14 +26,11 @@ phandle prom_chosen_node; * routines in the prom library. * It gets passed the pointer to the PROM vector. */ - -extern void prom_cif_init(void *, void *); - -void __init prom_init(void *cif_handler, void *cif_stack) +void __init prom_init(void *cif_handler) { phandle node; - prom_cif_init(cif_handler, cif_stack); + prom_cif_init(cif_handler); prom_chosen_node = prom_finddevice(prom_chosen_path); if (!prom_chosen_node || (s32)prom_chosen_node == -1) diff --git a/arch/sparc/prom/memory.c b/arch/sparc/prom/memory.c index 3f263a64857d..269d6ab5ef5e 100644 --- a/arch/sparc/prom/memory.c +++ b/arch/sparc/prom/memory.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* memory.c: Prom routine for acquiring various bits of information * about RAM on the machine, both virtual and physical. * diff --git a/arch/sparc/prom/misc_32.c b/arch/sparc/prom/misc_32.c index 8dc0b6b271e8..625750924860 100644 --- a/arch/sparc/prom/misc_32.c +++ b/arch/sparc/prom/misc_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * misc.c: Miscellaneous prom functions that don't belong * anywhere else. diff --git a/arch/sparc/prom/misc_64.c b/arch/sparc/prom/misc_64.c index f178b9dcc7b7..3792736ff21f 100644 --- a/arch/sparc/prom/misc_64.c +++ b/arch/sparc/prom/misc_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * misc.c: Miscellaneous prom functions that don't belong * anywhere else. @@ -81,11 +82,6 @@ void prom_feval(const char *fstring) } EXPORT_SYMBOL(prom_feval); -#ifdef CONFIG_SMP -extern void smp_capture(void); -extern void smp_release(void); -#endif - /* Drop into the prom, with the chance to continue with the 'go' * prom command. */ @@ -166,7 +162,7 @@ unsigned char prom_get_idprom(char *idbuf, int num_bytes) return 0xff; } -int prom_get_mmu_ihandle(void) +static int prom_get_mmu_ihandle(void) { phandle node; int ret; diff --git a/arch/sparc/prom/mp.c b/arch/sparc/prom/mp.c index 0da8256cf76f..67cf0e957e38 100644 --- a/arch/sparc/prom/mp.c +++ b/arch/sparc/prom/mp.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * mp.c: OpenBoot Prom Multiprocessor support routines. Don't call * these on a UP or else you will halt and catch fire. ;) diff --git a/arch/sparc/prom/p1275.c b/arch/sparc/prom/p1275.c index 04a4540509dd..51c3f984bbf7 100644 --- a/arch/sparc/prom/p1275.c +++ b/arch/sparc/prom/p1275.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * p1275.c: Sun IEEE 1275 PROM low level interface routines * @@ -5,11 +6,11 @@ */ #include <linux/kernel.h> -#include <linux/init.h> #include <linux/sched.h> #include <linux/smp.h> #include <linux/string.h> #include <linux/spinlock.h> +#include <linux/irqflags.h> #include <asm/openprom.h> #include <asm/oplib.h> @@ -20,7 +21,6 @@ struct { long prom_callback; /* 0x00 */ void (*prom_cif_handler)(long *); /* 0x08 */ - unsigned long prom_cif_stack; /* 0x10 */ } p1275buf; extern void prom_world(int); @@ -37,8 +37,8 @@ void p1275_cmd_direct(unsigned long *args) { unsigned long flags; - raw_local_save_flags(flags); - raw_local_irq_restore((unsigned long)PIL_NMI); + local_save_flags(flags); + local_irq_restore((unsigned long)PIL_NMI); raw_spin_lock(&prom_entry_lock); prom_world(1); @@ -46,11 +46,10 @@ void p1275_cmd_direct(unsigned long *args) prom_world(0); raw_spin_unlock(&prom_entry_lock); - raw_local_irq_restore(flags); + local_irq_restore(flags); } -void prom_cif_init(void *cif_handler, void *cif_stack) +void prom_cif_init(void *cif_handler) { p1275buf.prom_cif_handler = (void (*)(long *))cif_handler; - p1275buf.prom_cif_stack = (unsigned long)cif_stack; } diff --git a/arch/sparc/prom/printf.c b/arch/sparc/prom/printf.c index d9682f06b3b0..dcee3dfa6de9 100644 --- a/arch/sparc/prom/printf.c +++ b/arch/sparc/prom/printf.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * printf.c: Internal prom library printf facility. * diff --git a/arch/sparc/prom/ranges.c b/arch/sparc/prom/ranges.c index ad143c13bdc0..20cb828bc5f4 100644 --- a/arch/sparc/prom/ranges.c +++ b/arch/sparc/prom/ranges.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * ranges.c: Handle ranges in newer proms for obio/sbus. * @@ -16,9 +17,8 @@ static struct linux_prom_ranges promlib_obio_ranges[PROMREG_MAX]; static int num_obio_ranges; /* Adjust register values based upon the ranges parameters. */ -static void -prom_adjust_regs(struct linux_prom_registers *regp, int nregs, - struct linux_prom_ranges *rangep, int nranges) +static void prom_adjust_regs(struct linux_prom_registers *regp, int nregs, + struct linux_prom_ranges *rangep, int nranges) { int regc, rngc; @@ -34,33 +34,30 @@ prom_adjust_regs(struct linux_prom_registers *regp, int nregs, } } -static void -prom_adjust_ranges(struct linux_prom_ranges *ranges1, int nranges1, - struct linux_prom_ranges *ranges2, int nranges2) +static void prom_adjust_ranges(struct linux_prom_ranges *ranges1, int nranges1, + struct linux_prom_ranges *ranges2, int nranges2) { int rng1c, rng2c; - for(rng1c=0; rng1c < nranges1; rng1c++) { - for(rng2c=0; rng2c < nranges2; rng2c++) - if(ranges1[rng1c].ot_parent_space == ranges2[rng2c].ot_child_space && + for (rng1c = 0; rng1c < nranges1; rng1c++) { + for (rng2c = 0; rng2c < nranges2; rng2c++) + if (ranges1[rng1c].ot_parent_space == ranges2[rng2c].ot_child_space && ranges1[rng1c].ot_parent_base >= ranges2[rng2c].ot_child_base && ranges2[rng2c].ot_child_base + ranges2[rng2c].or_size - ranges1[rng1c].ot_parent_base > 0U) break; - if(rng2c == nranges2) /* oops */ + if (rng2c == nranges2) /* oops */ prom_printf("adjust_ranges: Could not find matching bus type...\n"); else if (ranges1[rng1c].ot_parent_base + ranges1[rng1c].or_size > ranges2[rng2c].ot_child_base + ranges2[rng2c].or_size) - ranges1[rng1c].or_size = - ranges2[rng2c].ot_child_base + ranges2[rng2c].or_size - ranges1[rng1c].ot_parent_base; + ranges1[rng1c].or_size = ranges2[rng2c].ot_child_base + ranges2[rng2c].or_size - ranges1[rng1c].ot_parent_base; ranges1[rng1c].ot_parent_space = ranges2[rng2c].ot_parent_space; ranges1[rng1c].ot_parent_base += ranges2[rng2c].ot_parent_base; } } /* Apply probed obio ranges to registers passed, if no ranges return. */ -void -prom_apply_obio_ranges(struct linux_prom_registers *regs, int nregs) +void prom_apply_obio_ranges(struct linux_prom_registers *regs, int nregs) { - if(num_obio_ranges) + if (num_obio_ranges) prom_adjust_regs(regs, nregs, promlib_obio_ranges, num_obio_ranges); } EXPORT_SYMBOL(prom_apply_obio_ranges); @@ -76,40 +73,40 @@ void __init prom_ranges_init(void) node = prom_getchild(prom_root_node); obio_node = prom_searchsiblings(node, "obio"); - if(obio_node) { + if (obio_node) { success = prom_getproperty(obio_node, "ranges", (char *) promlib_obio_ranges, sizeof(promlib_obio_ranges)); - if(success != -1) - num_obio_ranges = (success/sizeof(struct linux_prom_ranges)); + if (success != -1) + num_obio_ranges = (success / sizeof(struct linux_prom_ranges)); } - if(num_obio_ranges) + if (num_obio_ranges) prom_printf("PROMLIB: obio_ranges %d\n", num_obio_ranges); } void prom_apply_generic_ranges(phandle node, phandle parent, - struct linux_prom_registers *regs, int nregs) + struct linux_prom_registers *regs, int nregs) { int success; int num_ranges; struct linux_prom_ranges ranges[PROMREG_MAX]; - + success = prom_getproperty(node, "ranges", (char *) ranges, - sizeof (ranges)); + sizeof(ranges)); if (success != -1) { - num_ranges = (success/sizeof(struct linux_prom_ranges)); + num_ranges = (success / sizeof(struct linux_prom_ranges)); if (parent) { struct linux_prom_ranges parent_ranges[PROMREG_MAX]; int num_parent_ranges; - + success = prom_getproperty(parent, "ranges", - (char *) parent_ranges, - sizeof (parent_ranges)); + (char *) parent_ranges, + sizeof(parent_ranges)); if (success != -1) { - num_parent_ranges = (success/sizeof(struct linux_prom_ranges)); - prom_adjust_ranges (ranges, num_ranges, parent_ranges, num_parent_ranges); + num_parent_ranges = (success / sizeof(struct linux_prom_ranges)); + prom_adjust_ranges(ranges, num_ranges, parent_ranges, num_parent_ranges); } } prom_adjust_regs(regs, nregs, ranges, num_ranges); diff --git a/arch/sparc/prom/tree_32.c b/arch/sparc/prom/tree_32.c index f30e8d038f01..0fed89375b74 100644 --- a/arch/sparc/prom/tree_32.c +++ b/arch/sparc/prom/tree_32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * tree.c: Basic device tree traversal/scanning for the Linux * prom library. diff --git a/arch/sparc/prom/tree_64.c b/arch/sparc/prom/tree_64.c index bd1b2a3ac34e..7388339bbd7e 100644 --- a/arch/sparc/prom/tree_64.c +++ b/arch/sparc/prom/tree_64.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * tree.c: Basic device tree traversal/scanning for the Linux * prom library. @@ -271,7 +272,7 @@ char *prom_nextprop(phandle node, const char *oprop, char *buffer) return buffer; } if (oprop == buffer) { - strcpy (buf, oprop); + strscpy(buf, oprop); oprop = buf; } @@ -331,7 +332,7 @@ prom_setprop(phandle node, const char *pname, char *value, int size) if (size == 0) return 0; - if ((pname == 0) || (value == 0)) + if ((pname == NULL) || (value == NULL)) return 0; #ifdef CONFIG_SUN_LDOMS diff --git a/arch/sparc/vdso/.gitignore b/arch/sparc/vdso/.gitignore new file mode 100644 index 000000000000..8d4ebc990bf3 --- /dev/null +++ b/arch/sparc/vdso/.gitignore @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +vdso.lds +vdso-image-*.c +vdso2c diff --git a/arch/sparc/vdso/Makefile b/arch/sparc/vdso/Makefile new file mode 100644 index 000000000000..683b2d408224 --- /dev/null +++ b/arch/sparc/vdso/Makefile @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Building vDSO images for sparc. +# + +# files to link into the vdso +vobjs-y := vdso-note.o vclock_gettime.o + +# files to link into kernel +obj-y += vma.o + +# vDSO images to build +obj-$(CONFIG_SPARC64) += vdso-image-64.o +obj-$(CONFIG_COMPAT) += vdso-image-32.o + +vobjs := $(addprefix $(obj)/, $(vobjs-y)) + +$(obj)/vdso.o: $(obj)/vdso.so + +targets += vdso.lds $(vobjs-y) +targets += $(foreach x, 32 64, vdso-image-$(x).c vdso$(x).so vdso$(x).so.dbg) + +CPPFLAGS_vdso.lds += -P -C + +VDSO_LDFLAGS_vdso.lds = -m elf64_sparc -soname linux-vdso.so.1 \ + -z max-page-size=8192 + +$(obj)/vdso64.so.dbg: $(obj)/vdso.lds $(vobjs) FORCE + $(call if_changed,vdso) + +HOST_EXTRACFLAGS += -I$(srctree)/tools/include +hostprogs += vdso2c + +quiet_cmd_vdso2c = VDSO2C $@ + cmd_vdso2c = $(obj)/vdso2c $< $(<:%.dbg=%) $@ + +$(obj)/vdso-image-%.c: $(obj)/vdso%.so.dbg $(obj)/vdso%.so $(obj)/vdso2c FORCE + $(call if_changed,vdso2c) + +# +# Don't omit frame pointers for ease of userspace debugging, but do +# optimize sibling calls. +# +CFL := $(PROFILING) -mcmodel=medlow -fPIC -O2 -fasynchronous-unwind-tables -m64 \ + $(filter -g%,$(KBUILD_CFLAGS)) -fno-stack-protector \ + -fno-omit-frame-pointer -foptimize-sibling-calls \ + -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO + +SPARC_REG_CFLAGS = -ffixed-g4 -ffixed-g5 $(call cc-option,-fcall-used-g5) $(call cc-option,-fcall-used-g7) + +$(vobjs): KBUILD_CFLAGS := $(filter-out $(RANDSTRUCT_CFLAGS) $(KSTACK_ERASE_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(SPARC_REG_CFLAGS),$(KBUILD_CFLAGS)) $(CFL) + +# +# vDSO code runs in userspace and -pg doesn't help with profiling anyway. +# +CFLAGS_REMOVE_vclock_gettime.o = -pg +CFLAGS_REMOVE_vdso32/vclock_gettime.o = -pg + +$(obj)/%.so: OBJCOPYFLAGS := -S +$(obj)/%.so: $(obj)/%.so.dbg FORCE + $(call if_changed,objcopy) + +CPPFLAGS_vdso32/vdso32.lds = $(CPPFLAGS_vdso.lds) +VDSO_LDFLAGS_vdso32.lds = -m elf32_sparc -soname linux-gate.so.1 + +#This makes sure the $(obj) subdirectory exists even though vdso32/ +#is not a kbuild sub-make subdirectory +override obj-dirs = $(dir $(obj)) $(obj)/vdso32/ + +targets += vdso32/vdso32.lds +targets += vdso32/vdso-note.o +targets += vdso32/vclock_gettime.o + +KBUILD_AFLAGS_32 := $(filter-out -m64,$(KBUILD_AFLAGS)) -DBUILD_VDSO +$(obj)/vdso32.so.dbg: KBUILD_AFLAGS = $(KBUILD_AFLAGS_32) +$(obj)/vdso32.so.dbg: asflags-$(CONFIG_SPARC64) += -m32 + +KBUILD_CFLAGS_32 := $(filter-out -m64,$(KBUILD_CFLAGS)) +KBUILD_CFLAGS_32 := $(filter-out -mcmodel=medlow,$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out -fno-pic,$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out $(RANDSTRUCT_CFLAGS),$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out $(KSTACK_ERASE_CFLAGS),$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out $(SPARC_REG_CFLAGS),$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 += -m32 -msoft-float -fpic +KBUILD_CFLAGS_32 += -fno-stack-protector +KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls) +KBUILD_CFLAGS_32 += -fno-omit-frame-pointer +KBUILD_CFLAGS_32 += -DDISABLE_BRANCH_PROFILING +KBUILD_CFLAGS_32 += -mv8plus +$(obj)/vdso32.so.dbg: KBUILD_CFLAGS = $(KBUILD_CFLAGS_32) + +$(obj)/vdso32.so.dbg: FORCE \ + $(obj)/vdso32/vdso32.lds \ + $(obj)/vdso32/vclock_gettime.o \ + $(obj)/vdso32/vdso-note.o + $(call if_changed,vdso) + +# +# The DSO images are built using a special linker script. +# +quiet_cmd_vdso = VDSO $@ + cmd_vdso = $(LD) -nostdlib -o $@ \ + $(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \ + -T $(filter %.lds,$^) $(filter %.o,$^) + +VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic --no-undefined diff --git a/arch/sparc/vdso/vclock_gettime.c b/arch/sparc/vdso/vclock_gettime.c new file mode 100644 index 000000000000..79607804ea1b --- /dev/null +++ b/arch/sparc/vdso/vclock_gettime.c @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2006 Andi Kleen, SUSE Labs. + * + * Fast user context implementation of clock_gettime, gettimeofday, and time. + * + * The code should have no internal unresolved relocations. + * Check with readelf after changing. + * Also alternative() doesn't work. + */ +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#include <linux/kernel.h> +#include <linux/time.h> +#include <linux/string.h> +#include <asm/io.h> +#include <asm/unistd.h> +#include <asm/timex.h> +#include <asm/clocksource.h> +#include <asm/vvar.h> + +#ifdef CONFIG_SPARC64 +#define SYSCALL_STRING \ + "ta 0x6d;" \ + "bcs,a 1f;" \ + " sub %%g0, %%o0, %%o0;" \ + "1:" +#else +#define SYSCALL_STRING \ + "ta 0x10;" \ + "bcs,a 1f;" \ + " sub %%g0, %%o0, %%o0;" \ + "1:" +#endif + +#define SYSCALL_CLOBBERS \ + "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \ + "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", \ + "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", \ + "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", \ + "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", \ + "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", \ + "cc", "memory" + +/* + * Compute the vvar page's address in the process address space, and return it + * as a pointer to the vvar_data. + */ +notrace static __always_inline struct vvar_data *get_vvar_data(void) +{ + unsigned long ret; + + /* + * vdso data page is the first vDSO page so grab the PC + * and move up a page to get to the data page. + */ + __asm__("rd %%pc, %0" : "=r" (ret)); + ret &= ~(8192 - 1); + ret -= 8192; + + return (struct vvar_data *) ret; +} + +notrace static long vdso_fallback_gettime(long clock, struct __kernel_old_timespec *ts) +{ + register long num __asm__("g1") = __NR_clock_gettime; + register long o0 __asm__("o0") = clock; + register long o1 __asm__("o1") = (long) ts; + + __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), + "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); + return o0; +} + +notrace static long vdso_fallback_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + register long num __asm__("g1") = __NR_gettimeofday; + register long o0 __asm__("o0") = (long) tv; + register long o1 __asm__("o1") = (long) tz; + + __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), + "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); + return o0; +} + +#ifdef CONFIG_SPARC64 +notrace static __always_inline u64 __shr64(u64 val, int amt) +{ + return val >> amt; +} + +notrace static __always_inline u64 vread_tick(void) +{ + u64 ret; + + __asm__ __volatile__("rd %%tick, %0" : "=r" (ret)); + return ret; +} + +notrace static __always_inline u64 vread_tick_stick(void) +{ + u64 ret; + + __asm__ __volatile__("rd %%asr24, %0" : "=r" (ret)); + return ret; +} +#else +notrace static __always_inline u64 __shr64(u64 val, int amt) +{ + u64 ret; + + __asm__ __volatile__("sllx %H1, 32, %%g1\n\t" + "srl %L1, 0, %L1\n\t" + "or %%g1, %L1, %%g1\n\t" + "srlx %%g1, %2, %L0\n\t" + "srlx %L0, 32, %H0" + : "=r" (ret) + : "r" (val), "r" (amt) + : "g1"); + return ret; +} + +notrace static __always_inline u64 vread_tick(void) +{ + register unsigned long long ret asm("o4"); + + __asm__ __volatile__("rd %%tick, %L0\n\t" + "srlx %L0, 32, %H0" + : "=r" (ret)); + return ret; +} + +notrace static __always_inline u64 vread_tick_stick(void) +{ + register unsigned long long ret asm("o4"); + + __asm__ __volatile__("rd %%asr24, %L0\n\t" + "srlx %L0, 32, %H0" + : "=r" (ret)); + return ret; +} +#endif + +notrace static __always_inline u64 vgetsns(struct vvar_data *vvar) +{ + u64 v; + u64 cycles; + + cycles = vread_tick(); + v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask; + return v * vvar->clock.mult; +} + +notrace static __always_inline u64 vgetsns_stick(struct vvar_data *vvar) +{ + u64 v; + u64 cycles; + + cycles = vread_tick_stick(); + v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask; + return v * vvar->clock.mult; +} + +notrace static __always_inline int do_realtime(struct vvar_data *vvar, + struct __kernel_old_timespec *ts) +{ + unsigned long seq; + u64 ns; + + do { + seq = vvar_read_begin(vvar); + ts->tv_sec = vvar->wall_time_sec; + ns = vvar->wall_time_snsec; + ns += vgetsns(vvar); + ns = __shr64(ns, vvar->clock.shift); + } while (unlikely(vvar_read_retry(vvar, seq))); + + ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); + ts->tv_nsec = ns; + + return 0; +} + +notrace static __always_inline int do_realtime_stick(struct vvar_data *vvar, + struct __kernel_old_timespec *ts) +{ + unsigned long seq; + u64 ns; + + do { + seq = vvar_read_begin(vvar); + ts->tv_sec = vvar->wall_time_sec; + ns = vvar->wall_time_snsec; + ns += vgetsns_stick(vvar); + ns = __shr64(ns, vvar->clock.shift); + } while (unlikely(vvar_read_retry(vvar, seq))); + + ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); + ts->tv_nsec = ns; + + return 0; +} + +notrace static __always_inline int do_monotonic(struct vvar_data *vvar, + struct __kernel_old_timespec *ts) +{ + unsigned long seq; + u64 ns; + + do { + seq = vvar_read_begin(vvar); + ts->tv_sec = vvar->monotonic_time_sec; + ns = vvar->monotonic_time_snsec; + ns += vgetsns(vvar); + ns = __shr64(ns, vvar->clock.shift); + } while (unlikely(vvar_read_retry(vvar, seq))); + + ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); + ts->tv_nsec = ns; + + return 0; +} + +notrace static __always_inline int do_monotonic_stick(struct vvar_data *vvar, + struct __kernel_old_timespec *ts) +{ + unsigned long seq; + u64 ns; + + do { + seq = vvar_read_begin(vvar); + ts->tv_sec = vvar->monotonic_time_sec; + ns = vvar->monotonic_time_snsec; + ns += vgetsns_stick(vvar); + ns = __shr64(ns, vvar->clock.shift); + } while (unlikely(vvar_read_retry(vvar, seq))); + + ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); + ts->tv_nsec = ns; + + return 0; +} + +notrace static int do_realtime_coarse(struct vvar_data *vvar, + struct __kernel_old_timespec *ts) +{ + unsigned long seq; + + do { + seq = vvar_read_begin(vvar); + ts->tv_sec = vvar->wall_time_coarse_sec; + ts->tv_nsec = vvar->wall_time_coarse_nsec; + } while (unlikely(vvar_read_retry(vvar, seq))); + return 0; +} + +notrace static int do_monotonic_coarse(struct vvar_data *vvar, + struct __kernel_old_timespec *ts) +{ + unsigned long seq; + + do { + seq = vvar_read_begin(vvar); + ts->tv_sec = vvar->monotonic_time_coarse_sec; + ts->tv_nsec = vvar->monotonic_time_coarse_nsec; + } while (unlikely(vvar_read_retry(vvar, seq))); + + return 0; +} + +notrace int +__vdso_clock_gettime(clockid_t clock, struct __kernel_old_timespec *ts) +{ + struct vvar_data *vvd = get_vvar_data(); + + switch (clock) { + case CLOCK_REALTIME: + if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) + break; + return do_realtime(vvd, ts); + case CLOCK_MONOTONIC: + if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) + break; + return do_monotonic(vvd, ts); + case CLOCK_REALTIME_COARSE: + return do_realtime_coarse(vvd, ts); + case CLOCK_MONOTONIC_COARSE: + return do_monotonic_coarse(vvd, ts); + } + /* + * Unknown clock ID ? Fall back to the syscall. + */ + return vdso_fallback_gettime(clock, ts); +} +int +clock_gettime(clockid_t, struct __kernel_old_timespec *) + __attribute__((weak, alias("__vdso_clock_gettime"))); + +notrace int +__vdso_clock_gettime_stick(clockid_t clock, struct __kernel_old_timespec *ts) +{ + struct vvar_data *vvd = get_vvar_data(); + + switch (clock) { + case CLOCK_REALTIME: + if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) + break; + return do_realtime_stick(vvd, ts); + case CLOCK_MONOTONIC: + if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) + break; + return do_monotonic_stick(vvd, ts); + case CLOCK_REALTIME_COARSE: + return do_realtime_coarse(vvd, ts); + case CLOCK_MONOTONIC_COARSE: + return do_monotonic_coarse(vvd, ts); + } + /* + * Unknown clock ID ? Fall back to the syscall. + */ + return vdso_fallback_gettime(clock, ts); +} + +notrace int +__vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + struct vvar_data *vvd = get_vvar_data(); + + if (likely(vvd->vclock_mode != VCLOCK_NONE)) { + if (likely(tv != NULL)) { + union tstv_t { + struct __kernel_old_timespec ts; + struct __kernel_old_timeval tv; + } *tstv = (union tstv_t *) tv; + do_realtime(vvd, &tstv->ts); + /* + * Assign before dividing to ensure that the division is + * done in the type of tv_usec, not tv_nsec. + * + * There cannot be > 1 billion usec in a second: + * do_realtime() has already distributed such overflow + * into tv_sec. So we can assign it to an int safely. + */ + tstv->tv.tv_usec = tstv->ts.tv_nsec; + tstv->tv.tv_usec /= 1000; + } + if (unlikely(tz != NULL)) { + /* Avoid memcpy. Some old compilers fail to inline it */ + tz->tz_minuteswest = vvd->tz_minuteswest; + tz->tz_dsttime = vvd->tz_dsttime; + } + return 0; + } + return vdso_fallback_gettimeofday(tv, tz); +} +int +gettimeofday(struct __kernel_old_timeval *, struct timezone *) + __attribute__((weak, alias("__vdso_gettimeofday"))); + +notrace int +__vdso_gettimeofday_stick(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + struct vvar_data *vvd = get_vvar_data(); + + if (likely(vvd->vclock_mode != VCLOCK_NONE)) { + if (likely(tv != NULL)) { + union tstv_t { + struct __kernel_old_timespec ts; + struct __kernel_old_timeval tv; + } *tstv = (union tstv_t *) tv; + do_realtime_stick(vvd, &tstv->ts); + /* + * Assign before dividing to ensure that the division is + * done in the type of tv_usec, not tv_nsec. + * + * There cannot be > 1 billion usec in a second: + * do_realtime() has already distributed such overflow + * into tv_sec. So we can assign it to an int safely. + */ + tstv->tv.tv_usec = tstv->ts.tv_nsec; + tstv->tv.tv_usec /= 1000; + } + if (unlikely(tz != NULL)) { + /* Avoid memcpy. Some old compilers fail to inline it */ + tz->tz_minuteswest = vvd->tz_minuteswest; + tz->tz_dsttime = vvd->tz_dsttime; + } + return 0; + } + return vdso_fallback_gettimeofday(tv, tz); +} diff --git a/arch/sparc/vdso/vdso-layout.lds.S b/arch/sparc/vdso/vdso-layout.lds.S new file mode 100644 index 000000000000..d31e57e8a3bb --- /dev/null +++ b/arch/sparc/vdso/vdso-layout.lds.S @@ -0,0 +1,98 @@ +/* + * Linker script for vDSO. This is an ELF shared object prelinked to + * its virtual address, and with only one read-only segment. + * This script controls its layout. + */ + +#if defined(BUILD_VDSO64) +# define SHDR_SIZE 64 +#elif defined(BUILD_VDSO32) +# define SHDR_SIZE 40 +#else +# error unknown VDSO target +#endif + +#define NUM_FAKE_SHDRS 7 + +SECTIONS +{ + /* + * User/kernel shared data is before the vDSO. This may be a little + * uglier than putting it after the vDSO, but it avoids issues with + * non-allocatable things that dangle past the end of the PT_LOAD + * segment. Page size is 8192 for both 64-bit and 32-bit vdso binaries + */ + + vvar_start = . -8192; + vvar_data = vvar_start; + + . = SIZEOF_HEADERS; + + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .gnu.version : { *(.gnu.version) } + .gnu.version_d : { *(.gnu.version_d) } + .gnu.version_r : { *(.gnu.version_r) } + + .dynamic : { *(.dynamic) } :text :dynamic + + .rodata : { + *(.rodata*) + *(.data*) + *(.sdata*) + *(.got.plt) *(.got) + *(.gnu.linkonce.d.*) + *(.bss*) + *(.dynbss*) + *(.gnu.linkonce.b.*) + + /* + * Ideally this would live in a C file: kept in here for + * compatibility with x86-64. + */ + VDSO_FAKE_SECTION_TABLE_START = .; + . = . + NUM_FAKE_SHDRS * SHDR_SIZE; + VDSO_FAKE_SECTION_TABLE_END = .; + } :text + + .fake_shstrtab : { *(.fake_shstrtab) } :text + + + .note : { *(.note.*) } :text :note + + .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr + .eh_frame : { KEEP (*(.eh_frame)) } :text + + + /* + * Text is well-separated from actual data: there's plenty of + * stuff that isn't used at runtime in between. + */ + + .text : { *(.text*) } :text =0x90909090, + + /DISCARD/ : { + *(.discard) + *(.discard.*) + *(__bug_table) + } +} + +/* + * Very old versions of ld do not recognize this name token; use the constant. + */ +#define PT_GNU_EH_FRAME 0x6474e550 + +/* + * We must supply the ELF program headers explicitly to get just one + * PT_LOAD segment, and set the flags explicitly to make segments read-only. + */ +PHDRS +{ + text PT_LOAD FLAGS(5) FILEHDR PHDRS; /* PF_R|PF_X */ + dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ + note PT_NOTE FLAGS(4); /* PF_R */ + eh_frame_hdr PT_GNU_EH_FRAME; +} diff --git a/arch/sparc/vdso/vdso-note.S b/arch/sparc/vdso/vdso-note.S new file mode 100644 index 000000000000..79a071e4357e --- /dev/null +++ b/arch/sparc/vdso/vdso-note.S @@ -0,0 +1,12 @@ +/* + * This supplies .note.* sections to go into the PT_NOTE inside the vDSO text. + * Here we can supply some information useful to userland. + */ + +#include <linux/uts.h> +#include <linux/version.h> +#include <linux/elfnote.h> + +ELFNOTE_START(Linux, 0, "a") + .long LINUX_VERSION_CODE +ELFNOTE_END diff --git a/arch/sparc/vdso/vdso.lds.S b/arch/sparc/vdso/vdso.lds.S new file mode 100644 index 000000000000..629ab6900df7 --- /dev/null +++ b/arch/sparc/vdso/vdso.lds.S @@ -0,0 +1,27 @@ +/* + * Linker script for 64-bit vDSO. + * We #include the file to define the layout details. + * + * This file defines the version script giving the user-exported symbols in + * the DSO. + */ + +#define BUILD_VDSO64 + +#include "vdso-layout.lds.S" + +/* + * This controls what userland symbols we export from the vDSO. + */ +VERSION { + LINUX_2.6 { + global: + clock_gettime; + __vdso_clock_gettime; + __vdso_clock_gettime_stick; + gettimeofday; + __vdso_gettimeofday; + __vdso_gettimeofday_stick; + local: *; + }; +} diff --git a/arch/sparc/vdso/vdso2c.c b/arch/sparc/vdso/vdso2c.c new file mode 100644 index 000000000000..dc81240aab6f --- /dev/null +++ b/arch/sparc/vdso/vdso2c.c @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * vdso2c - A vdso image preparation tool + * Copyright (c) 2014 Andy Lutomirski and others + * + * vdso2c requires stripped and unstripped input. It would be trivial + * to fully strip the input in here, but, for reasons described below, + * we need to write a section table. Doing this is more or less + * equivalent to dropping all non-allocatable sections, but it's + * easier to let objcopy handle that instead of doing it ourselves. + * If we ever need to do something fancier than what objcopy provides, + * it would be straightforward to add here. + * + * We keep a section table for a few reasons: + * + * Binutils has issues debugging the vDSO: it reads the section table to + * find SHT_NOTE; it won't look at PT_NOTE for the in-memory vDSO, which + * would break build-id if we removed the section table. Binutils + * also requires that shstrndx != 0. See: + * https://sourceware.org/bugzilla/show_bug.cgi?id=17064 + * + * elfutils might not look for PT_NOTE if there is a section table at + * all. I don't know whether this matters for any practical purpose. + * + * For simplicity, rather than hacking up a partial section table, we + * just write a mostly complete one. We omit non-dynamic symbols, + * though, since they're rather large. + * + * Once binutils gets fixed, we might be able to drop this for all but + * the 64-bit vdso, since build-id only works in kernel RPMs, and + * systems that update to new enough kernel RPMs will likely update + * binutils in sync. build-id has never worked for home-built kernel + * RPMs without manual symlinking, and I suspect that no one ever does + * that. + */ + +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#include <inttypes.h> +#include <stdint.h> +#include <unistd.h> +#include <stdarg.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <fcntl.h> +#include <err.h> + +#include <sys/mman.h> +#include <sys/types.h> +#include <tools/be_byteshift.h> + +#include <linux/elf.h> +#include <linux/types.h> +#include <linux/kernel.h> + +const char *outfilename; + +/* Symbols that we need in vdso2c. */ +enum { + sym_vvar_start, + sym_VDSO_FAKE_SECTION_TABLE_START, + sym_VDSO_FAKE_SECTION_TABLE_END, +}; + +struct vdso_sym { + const char *name; + int export; +}; + +struct vdso_sym required_syms[] = { + [sym_vvar_start] = {"vvar_start", 1}, + [sym_VDSO_FAKE_SECTION_TABLE_START] = { + "VDSO_FAKE_SECTION_TABLE_START", 0 + }, + [sym_VDSO_FAKE_SECTION_TABLE_END] = { + "VDSO_FAKE_SECTION_TABLE_END", 0 + }, +}; + +__attribute__((format(printf, 1, 2))) __attribute__((noreturn)) +static void fail(const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + fprintf(stderr, "Error: "); + vfprintf(stderr, format, ap); + if (outfilename) + unlink(outfilename); + exit(1); + va_end(ap); +} + +/* + * Evil macros for big-endian reads and writes + */ +#define GBE(x, bits, ifnot) \ + __builtin_choose_expr( \ + (sizeof(*(x)) == bits/8), \ + (__typeof__(*(x)))get_unaligned_be##bits(x), ifnot) + +#define LAST_GBE(x) \ + __builtin_choose_expr(sizeof(*(x)) == 1, *(x), (void)(0)) + +#define GET_BE(x) \ + GBE(x, 64, GBE(x, 32, GBE(x, 16, LAST_GBE(x)))) + +#define PBE(x, val, bits, ifnot) \ + __builtin_choose_expr( \ + (sizeof(*(x)) == bits/8), \ + put_unaligned_be##bits((val), (x)), ifnot) + +#define LAST_PBE(x, val) \ + __builtin_choose_expr(sizeof(*(x)) == 1, *(x) = (val), (void)(0)) + +#define PUT_BE(x, val) \ + PBE(x, val, 64, PBE(x, val, 32, PBE(x, val, 16, LAST_PBE(x, val)))) + +#define NSYMS ARRAY_SIZE(required_syms) + +#define BITSFUNC3(name, bits, suffix) name##bits##suffix +#define BITSFUNC2(name, bits, suffix) BITSFUNC3(name, bits, suffix) +#define BITSFUNC(name) BITSFUNC2(name, ELF_BITS, ) + +#define INT_BITS BITSFUNC2(int, ELF_BITS, _t) + +#define ELF_BITS_XFORM2(bits, x) Elf##bits##_##x +#define ELF_BITS_XFORM(bits, x) ELF_BITS_XFORM2(bits, x) +#define ELF(x) ELF_BITS_XFORM(ELF_BITS, x) + +#define ELF_BITS 64 +#include "vdso2c.h" +#undef ELF_BITS + +#define ELF_BITS 32 +#include "vdso2c.h" +#undef ELF_BITS + +static void go(void *raw_addr, size_t raw_len, + void *stripped_addr, size_t stripped_len, + FILE *outfile, const char *name) +{ + Elf64_Ehdr *hdr = (Elf64_Ehdr *)raw_addr; + + if (hdr->e_ident[EI_CLASS] == ELFCLASS64) { + go64(raw_addr, raw_len, stripped_addr, stripped_len, + outfile, name); + } else if (hdr->e_ident[EI_CLASS] == ELFCLASS32) { + go32(raw_addr, raw_len, stripped_addr, stripped_len, + outfile, name); + } else { + fail("unknown ELF class\n"); + } +} + +static void map_input(const char *name, void **addr, size_t *len, int prot) +{ + off_t tmp_len; + + int fd = open(name, O_RDONLY); + + if (fd == -1) + err(1, "%s", name); + + tmp_len = lseek(fd, 0, SEEK_END); + if (tmp_len == (off_t)-1) + err(1, "lseek"); + *len = (size_t)tmp_len; + + *addr = mmap(NULL, tmp_len, prot, MAP_PRIVATE, fd, 0); + if (*addr == MAP_FAILED) + err(1, "mmap"); + + close(fd); +} + +int main(int argc, char **argv) +{ + size_t raw_len, stripped_len; + void *raw_addr, *stripped_addr; + FILE *outfile; + char *name, *tmp; + int namelen; + + if (argc != 4) { + printf("Usage: vdso2c RAW_INPUT STRIPPED_INPUT OUTPUT\n"); + return 1; + } + + /* + * Figure out the struct name. If we're writing to a .so file, + * generate raw output insted. + */ + name = strdup(argv[3]); + namelen = strlen(name); + if (namelen >= 3 && !strcmp(name + namelen - 3, ".so")) { + name = NULL; + } else { + tmp = strrchr(name, '/'); + if (tmp) + name = tmp + 1; + tmp = strchr(name, '.'); + if (tmp) + *tmp = '\0'; + for (tmp = name; *tmp; tmp++) + if (*tmp == '-') + *tmp = '_'; + } + + map_input(argv[1], &raw_addr, &raw_len, PROT_READ); + map_input(argv[2], &stripped_addr, &stripped_len, PROT_READ); + + outfilename = argv[3]; + outfile = fopen(outfilename, "w"); + if (!outfile) + err(1, "%s", argv[2]); + + go(raw_addr, raw_len, stripped_addr, stripped_len, outfile, name); + + munmap(raw_addr, raw_len); + munmap(stripped_addr, stripped_len); + fclose(outfile); + + return 0; +} diff --git a/arch/sparc/vdso/vdso2c.h b/arch/sparc/vdso/vdso2c.h new file mode 100644 index 000000000000..60d69acc748f --- /dev/null +++ b/arch/sparc/vdso/vdso2c.h @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +/* + * This file is included up to twice from vdso2c.c. It generates code for + * 32-bit and 64-bit vDSOs. We will eventually need both for 64-bit builds, + * since 32-bit vDSOs will then be built for 32-bit userspace. + */ + +static void BITSFUNC(go)(void *raw_addr, size_t raw_len, + void *stripped_addr, size_t stripped_len, + FILE *outfile, const char *name) +{ + int found_load = 0; + unsigned long load_size = -1; /* Work around bogus warning */ + unsigned long mapping_size; + int i; + unsigned long j; + ELF(Shdr) *symtab_hdr = NULL, *strtab_hdr; + ELF(Ehdr) *hdr = (ELF(Ehdr) *)raw_addr; + ELF(Dyn) *dyn = 0, *dyn_end = 0; + INT_BITS syms[NSYMS] = {}; + + ELF(Phdr) *pt = (ELF(Phdr) *)(raw_addr + GET_BE(&hdr->e_phoff)); + + /* Walk the segment table. */ + for (i = 0; i < GET_BE(&hdr->e_phnum); i++) { + if (GET_BE(&pt[i].p_type) == PT_LOAD) { + if (found_load) + fail("multiple PT_LOAD segs\n"); + + if (GET_BE(&pt[i].p_offset) != 0 || + GET_BE(&pt[i].p_vaddr) != 0) + fail("PT_LOAD in wrong place\n"); + + if (GET_BE(&pt[i].p_memsz) != GET_BE(&pt[i].p_filesz)) + fail("cannot handle memsz != filesz\n"); + + load_size = GET_BE(&pt[i].p_memsz); + found_load = 1; + } else if (GET_BE(&pt[i].p_type) == PT_DYNAMIC) { + dyn = raw_addr + GET_BE(&pt[i].p_offset); + dyn_end = raw_addr + GET_BE(&pt[i].p_offset) + + GET_BE(&pt[i].p_memsz); + } + } + if (!found_load) + fail("no PT_LOAD seg\n"); + + if (stripped_len < load_size) + fail("stripped input is too short\n"); + + /* Walk the dynamic table */ + for (i = 0; dyn + i < dyn_end && + GET_BE(&dyn[i].d_tag) != DT_NULL; i++) { + typeof(dyn[i].d_tag) tag = GET_BE(&dyn[i].d_tag); + typeof(dyn[i].d_un.d_val) val = GET_BE(&dyn[i].d_un.d_val); + + if ((tag == DT_RELSZ || tag == DT_RELASZ) && (val != 0)) + fail("vdso image contains dynamic relocations\n"); + } + + /* Walk the section table */ + for (i = 0; i < GET_BE(&hdr->e_shnum); i++) { + ELF(Shdr) *sh = raw_addr + GET_BE(&hdr->e_shoff) + + GET_BE(&hdr->e_shentsize) * i; + if (GET_BE(&sh->sh_type) == SHT_SYMTAB) + symtab_hdr = sh; + } + + if (!symtab_hdr) + fail("no symbol table\n"); + + strtab_hdr = raw_addr + GET_BE(&hdr->e_shoff) + + GET_BE(&hdr->e_shentsize) * GET_BE(&symtab_hdr->sh_link); + + /* Walk the symbol table */ + for (i = 0; + i < GET_BE(&symtab_hdr->sh_size) / GET_BE(&symtab_hdr->sh_entsize); + i++) { + int k; + + ELF(Sym) *sym = raw_addr + GET_BE(&symtab_hdr->sh_offset) + + GET_BE(&symtab_hdr->sh_entsize) * i; + const char *name = raw_addr + GET_BE(&strtab_hdr->sh_offset) + + GET_BE(&sym->st_name); + + for (k = 0; k < NSYMS; k++) { + if (!strcmp(name, required_syms[k].name)) { + if (syms[k]) { + fail("duplicate symbol %s\n", + required_syms[k].name); + } + + /* + * Careful: we use negative addresses, but + * st_value is unsigned, so we rely + * on syms[k] being a signed type of the + * correct width. + */ + syms[k] = GET_BE(&sym->st_value); + } + } + } + + /* Validate mapping addresses. */ + if (syms[sym_vvar_start] % 8192) + fail("vvar_begin must be a multiple of 8192\n"); + + if (!name) { + fwrite(stripped_addr, stripped_len, 1, outfile); + return; + } + + mapping_size = (stripped_len + 8191) / 8192 * 8192; + + fprintf(outfile, "/* AUTOMATICALLY GENERATED -- DO NOT EDIT */\n\n"); + fprintf(outfile, "#include <linux/cache.h>\n"); + fprintf(outfile, "#include <asm/vdso.h>\n"); + fprintf(outfile, "\n"); + fprintf(outfile, + "static unsigned char raw_data[%lu] __ro_after_init __aligned(8192)= {", + mapping_size); + for (j = 0; j < stripped_len; j++) { + if (j % 10 == 0) + fprintf(outfile, "\n\t"); + fprintf(outfile, "0x%02X, ", + (int)((unsigned char *)stripped_addr)[j]); + } + fprintf(outfile, "\n};\n\n"); + + fprintf(outfile, "const struct vdso_image %s_builtin = {\n", name); + fprintf(outfile, "\t.data = raw_data,\n"); + fprintf(outfile, "\t.size = %lu,\n", mapping_size); + for (i = 0; i < NSYMS; i++) { + if (required_syms[i].export && syms[i]) + fprintf(outfile, "\t.sym_%s = %" PRIi64 ",\n", + required_syms[i].name, (int64_t)syms[i]); + } + fprintf(outfile, "};\n"); +} diff --git a/arch/sparc/vdso/vdso32/.gitignore b/arch/sparc/vdso/vdso32/.gitignore new file mode 100644 index 000000000000..5167384843b9 --- /dev/null +++ b/arch/sparc/vdso/vdso32/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +vdso32.lds diff --git a/arch/sparc/vdso/vdso32/vclock_gettime.c b/arch/sparc/vdso/vdso32/vclock_gettime.c new file mode 100644 index 000000000000..d7f99e6745ea --- /dev/null +++ b/arch/sparc/vdso/vdso32/vclock_gettime.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#define BUILD_VDSO32 + +#ifdef CONFIG_SPARC64 + +/* + * in case of a 32 bit VDSO for a 64 bit kernel fake a 32 bit kernel + * configuration + */ +#undef CONFIG_64BIT +#undef CONFIG_SPARC64 +#define BUILD_VDSO32_64 +#define CONFIG_32BIT +#undef CONFIG_QUEUED_RWLOCKS +#undef CONFIG_QUEUED_SPINLOCKS + +#endif + +#include "../vclock_gettime.c" diff --git a/arch/sparc/vdso/vdso32/vdso-note.S b/arch/sparc/vdso/vdso32/vdso-note.S new file mode 100644 index 000000000000..e234983cf0d8 --- /dev/null +++ b/arch/sparc/vdso/vdso32/vdso-note.S @@ -0,0 +1,12 @@ +/* + * This supplies .note.* sections to go into the PT_NOTE inside the vDSO + * text. Here we can supply some information useful to userland. + */ + +#include <linux/uts.h> +#include <linux/version.h> +#include <linux/elfnote.h> + +ELFNOTE_START(Linux, 0, "a") + .long LINUX_VERSION_CODE +ELFNOTE_END diff --git a/arch/sparc/vdso/vdso32/vdso32.lds.S b/arch/sparc/vdso/vdso32/vdso32.lds.S new file mode 100644 index 000000000000..218930fdff03 --- /dev/null +++ b/arch/sparc/vdso/vdso32/vdso32.lds.S @@ -0,0 +1,26 @@ +/* + * Linker script for sparc32 vDSO + * We #include the file to define the layout details. + * + * This file defines the version script giving the user-exported symbols in + * the DSO. + */ + +#define BUILD_VDSO32 +#include "../vdso-layout.lds.S" + +/* + * This controls what userland symbols we export from the vDSO. + */ +VERSION { + LINUX_2.6 { + global: + clock_gettime; + __vdso_clock_gettime; + __vdso_clock_gettime_stick; + gettimeofday; + __vdso_gettimeofday; + __vdso_gettimeofday_stick; + local: *; + }; +} diff --git a/arch/sparc/vdso/vma.c b/arch/sparc/vdso/vma.c new file mode 100644 index 000000000000..bab7a59575e8 --- /dev/null +++ b/arch/sparc/vdso/vma.c @@ -0,0 +1,457 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Set up the VMAs to tell the VM about the vDSO. + * Copyright 2007 Andi Kleen, SUSE Labs. + */ + +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#include <linux/mm.h> +#include <linux/err.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/init.h> +#include <linux/linkage.h> +#include <linux/random.h> +#include <linux/elf.h> +#include <asm/cacheflush.h> +#include <asm/spitfire.h> +#include <asm/vdso.h> +#include <asm/vvar.h> +#include <asm/page.h> + +unsigned int __read_mostly vdso_enabled = 1; + +static struct vm_special_mapping vvar_mapping = { + .name = "[vvar]" +}; + +#ifdef CONFIG_SPARC64 +static struct vm_special_mapping vdso_mapping64 = { + .name = "[vdso]" +}; +#endif + +#ifdef CONFIG_COMPAT +static struct vm_special_mapping vdso_mapping32 = { + .name = "[vdso]" +}; +#endif + +struct vvar_data *vvar_data; + +struct vdso_elfinfo32 { + Elf32_Ehdr *hdr; + Elf32_Sym *dynsym; + unsigned long dynsymsize; + const char *dynstr; + unsigned long text; +}; + +struct vdso_elfinfo64 { + Elf64_Ehdr *hdr; + Elf64_Sym *dynsym; + unsigned long dynsymsize; + const char *dynstr; + unsigned long text; +}; + +struct vdso_elfinfo { + union { + struct vdso_elfinfo32 elf32; + struct vdso_elfinfo64 elf64; + } u; +}; + +static void *one_section64(struct vdso_elfinfo64 *e, const char *name, + unsigned long *size) +{ + const char *snames; + Elf64_Shdr *shdrs; + unsigned int i; + + shdrs = (void *)e->hdr + e->hdr->e_shoff; + snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset; + for (i = 1; i < e->hdr->e_shnum; i++) { + if (!strcmp(snames+shdrs[i].sh_name, name)) { + if (size) + *size = shdrs[i].sh_size; + return (void *)e->hdr + shdrs[i].sh_offset; + } + } + return NULL; +} + +static int find_sections64(const struct vdso_image *image, struct vdso_elfinfo *_e) +{ + struct vdso_elfinfo64 *e = &_e->u.elf64; + + e->hdr = image->data; + e->dynsym = one_section64(e, ".dynsym", &e->dynsymsize); + e->dynstr = one_section64(e, ".dynstr", NULL); + + if (!e->dynsym || !e->dynstr) { + pr_err("VDSO64: Missing symbol sections.\n"); + return -ENODEV; + } + return 0; +} + +static Elf64_Sym *find_sym64(const struct vdso_elfinfo64 *e, const char *name) +{ + unsigned int i; + + for (i = 0; i < (e->dynsymsize / sizeof(Elf64_Sym)); i++) { + Elf64_Sym *s = &e->dynsym[i]; + if (s->st_name == 0) + continue; + if (!strcmp(e->dynstr + s->st_name, name)) + return s; + } + return NULL; +} + +static int patchsym64(struct vdso_elfinfo *_e, const char *orig, + const char *new) +{ + struct vdso_elfinfo64 *e = &_e->u.elf64; + Elf64_Sym *osym = find_sym64(e, orig); + Elf64_Sym *nsym = find_sym64(e, new); + + if (!nsym || !osym) { + pr_err("VDSO64: Missing symbols.\n"); + return -ENODEV; + } + osym->st_value = nsym->st_value; + osym->st_size = nsym->st_size; + osym->st_info = nsym->st_info; + osym->st_other = nsym->st_other; + osym->st_shndx = nsym->st_shndx; + + return 0; +} + +static void *one_section32(struct vdso_elfinfo32 *e, const char *name, + unsigned long *size) +{ + const char *snames; + Elf32_Shdr *shdrs; + unsigned int i; + + shdrs = (void *)e->hdr + e->hdr->e_shoff; + snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset; + for (i = 1; i < e->hdr->e_shnum; i++) { + if (!strcmp(snames+shdrs[i].sh_name, name)) { + if (size) + *size = shdrs[i].sh_size; + return (void *)e->hdr + shdrs[i].sh_offset; + } + } + return NULL; +} + +static int find_sections32(const struct vdso_image *image, struct vdso_elfinfo *_e) +{ + struct vdso_elfinfo32 *e = &_e->u.elf32; + + e->hdr = image->data; + e->dynsym = one_section32(e, ".dynsym", &e->dynsymsize); + e->dynstr = one_section32(e, ".dynstr", NULL); + + if (!e->dynsym || !e->dynstr) { + pr_err("VDSO32: Missing symbol sections.\n"); + return -ENODEV; + } + return 0; +} + +static Elf32_Sym *find_sym32(const struct vdso_elfinfo32 *e, const char *name) +{ + unsigned int i; + + for (i = 0; i < (e->dynsymsize / sizeof(Elf32_Sym)); i++) { + Elf32_Sym *s = &e->dynsym[i]; + if (s->st_name == 0) + continue; + if (!strcmp(e->dynstr + s->st_name, name)) + return s; + } + return NULL; +} + +static int patchsym32(struct vdso_elfinfo *_e, const char *orig, + const char *new) +{ + struct vdso_elfinfo32 *e = &_e->u.elf32; + Elf32_Sym *osym = find_sym32(e, orig); + Elf32_Sym *nsym = find_sym32(e, new); + + if (!nsym || !osym) { + pr_err("VDSO32: Missing symbols.\n"); + return -ENODEV; + } + osym->st_value = nsym->st_value; + osym->st_size = nsym->st_size; + osym->st_info = nsym->st_info; + osym->st_other = nsym->st_other; + osym->st_shndx = nsym->st_shndx; + + return 0; +} + +static int find_sections(const struct vdso_image *image, struct vdso_elfinfo *e, + bool elf64) +{ + if (elf64) + return find_sections64(image, e); + else + return find_sections32(image, e); +} + +static int patch_one_symbol(struct vdso_elfinfo *e, const char *orig, + const char *new_target, bool elf64) +{ + if (elf64) + return patchsym64(e, orig, new_target); + else + return patchsym32(e, orig, new_target); +} + +static int stick_patch(const struct vdso_image *image, struct vdso_elfinfo *e, bool elf64) +{ + int err; + + err = find_sections(image, e, elf64); + if (err) + return err; + + err = patch_one_symbol(e, + "__vdso_gettimeofday", + "__vdso_gettimeofday_stick", elf64); + if (err) + return err; + + return patch_one_symbol(e, + "__vdso_clock_gettime", + "__vdso_clock_gettime_stick", elf64); + return 0; +} + +/* + * Allocate pages for the vdso and vvar, and copy in the vdso text from the + * kernel image. + */ +static int __init init_vdso_image(const struct vdso_image *image, + struct vm_special_mapping *vdso_mapping, + bool elf64) +{ + int cnpages = (image->size) / PAGE_SIZE; + struct page *dp, **dpp = NULL; + struct page *cp, **cpp = NULL; + struct vdso_elfinfo ei; + int i, dnpages = 0; + + if (tlb_type != spitfire) { + int err = stick_patch(image, &ei, elf64); + if (err) + return err; + } + + /* + * First, the vdso text. This is initialied data, an integral number of + * pages long. + */ + if (WARN_ON(image->size % PAGE_SIZE != 0)) + goto oom; + + cpp = kcalloc(cnpages, sizeof(struct page *), GFP_KERNEL); + vdso_mapping->pages = cpp; + + if (!cpp) + goto oom; + + for (i = 0; i < cnpages; i++) { + cp = alloc_page(GFP_KERNEL); + if (!cp) + goto oom; + cpp[i] = cp; + copy_page(page_address(cp), image->data + i * PAGE_SIZE); + } + + /* + * Now the vvar page. This is uninitialized data. + */ + + if (vvar_data == NULL) { + dnpages = (sizeof(struct vvar_data) / PAGE_SIZE) + 1; + if (WARN_ON(dnpages != 1)) + goto oom; + dpp = kcalloc(dnpages, sizeof(struct page *), GFP_KERNEL); + vvar_mapping.pages = dpp; + + if (!dpp) + goto oom; + + dp = alloc_page(GFP_KERNEL); + if (!dp) + goto oom; + + dpp[0] = dp; + vvar_data = page_address(dp); + memset(vvar_data, 0, PAGE_SIZE); + + vvar_data->seq = 0; + } + + return 0; + oom: + if (cpp != NULL) { + for (i = 0; i < cnpages; i++) { + if (cpp[i] != NULL) + __free_page(cpp[i]); + } + kfree(cpp); + vdso_mapping->pages = NULL; + } + + if (dpp != NULL) { + for (i = 0; i < dnpages; i++) { + if (dpp[i] != NULL) + __free_page(dpp[i]); + } + kfree(dpp); + vvar_mapping.pages = NULL; + } + + pr_warn("Cannot allocate vdso\n"); + vdso_enabled = 0; + return -ENOMEM; +} + +static int __init init_vdso(void) +{ + int err = 0; +#ifdef CONFIG_SPARC64 + err = init_vdso_image(&vdso_image_64_builtin, &vdso_mapping64, true); + if (err) + return err; +#endif + +#ifdef CONFIG_COMPAT + err = init_vdso_image(&vdso_image_32_builtin, &vdso_mapping32, false); +#endif + return err; + +} +subsys_initcall(init_vdso); + +struct linux_binprm; + +/* Shuffle the vdso up a bit, randomly. */ +static unsigned long vdso_addr(unsigned long start, unsigned int len) +{ + unsigned int offset; + + /* This loses some more bits than a modulo, but is cheaper */ + offset = get_random_u32_below(PTRS_PER_PTE); + return start + (offset << PAGE_SHIFT); +} + +static int map_vdso(const struct vdso_image *image, + struct vm_special_mapping *vdso_mapping) +{ + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + unsigned long text_start, addr = 0; + int ret = 0; + + mmap_write_lock(mm); + + /* + * First, get an unmapped region: then randomize it, and make sure that + * region is free. + */ + if (current->flags & PF_RANDOMIZE) { + addr = get_unmapped_area(NULL, 0, + image->size - image->sym_vvar_start, + 0, 0); + if (IS_ERR_VALUE(addr)) { + ret = addr; + goto up_fail; + } + addr = vdso_addr(addr, image->size - image->sym_vvar_start); + } + addr = get_unmapped_area(NULL, addr, + image->size - image->sym_vvar_start, 0, 0); + if (IS_ERR_VALUE(addr)) { + ret = addr; + goto up_fail; + } + + text_start = addr - image->sym_vvar_start; + current->mm->context.vdso = (void __user *)text_start; + + /* + * MAYWRITE to allow gdb to COW and set breakpoints + */ + vma = _install_special_mapping(mm, + text_start, + image->size, + VM_READ|VM_EXEC| + VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, + vdso_mapping); + + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + goto up_fail; + } + + vma = _install_special_mapping(mm, + addr, + -image->sym_vvar_start, + VM_READ|VM_MAYREAD, + &vvar_mapping); + + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + do_munmap(mm, text_start, image->size, NULL); + } + +up_fail: + if (ret) + current->mm->context.vdso = NULL; + + mmap_write_unlock(mm); + return ret; +} + +int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) +{ + + if (!vdso_enabled) + return 0; + +#if defined CONFIG_COMPAT + if (!(is_32bit_task())) + return map_vdso(&vdso_image_64_builtin, &vdso_mapping64); + else + return map_vdso(&vdso_image_32_builtin, &vdso_mapping32); +#else + return map_vdso(&vdso_image_64_builtin, &vdso_mapping64); +#endif + +} + +static __init int vdso_setup(char *s) +{ + int err; + unsigned long val; + + err = kstrtoul(s, 10, &val); + if (!err) + vdso_enabled = val; + return 1; +} +__setup("vdso=", vdso_setup); diff --git a/arch/sparc/video/Makefile b/arch/sparc/video/Makefile new file mode 100644 index 000000000000..dcfbe7a5912c --- /dev/null +++ b/arch/sparc/video/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-y += video-common.o diff --git a/arch/sparc/video/video-common.c b/arch/sparc/video/video-common.c new file mode 100644 index 000000000000..2414380caadc --- /dev/null +++ b/arch/sparc/video/video-common.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/console.h> +#include <linux/device.h> +#include <linux/module.h> + +#include <asm/prom.h> +#include <asm/video.h> + +bool video_is_primary_device(struct device *dev) +{ + struct device_node *node = dev->of_node; + + if (console_set_on_cmdline) + return false; + + if (node && node == of_console_device) + return true; + + return false; +} +EXPORT_SYMBOL(video_is_primary_device); + +MODULE_DESCRIPTION("Sparc video helpers"); +MODULE_LICENSE("GPL"); |