diff options
Diffstat (limited to 'arch/mips/kernel')
132 files changed, 22545 insertions, 13304 deletions
diff --git a/arch/mips/kernel/.gitignore b/arch/mips/kernel/.gitignore index c5f676c3c224..bbb90f92d051 100644 --- a/arch/mips/kernel/.gitignore +++ b/arch/mips/kernel/.gitignore @@ -1 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only vmlinux.lds diff --git a/arch/mips/kernel/8250-platform.c b/arch/mips/kernel/8250-platform.c deleted file mode 100644 index 5c6b2ab1f56e..000000000000 --- a/arch/mips/kernel/8250-platform.c +++ /dev/null @@ -1,46 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2007 Ralf Baechle (ralf@linux-mips.org) - */ -#include <linux/init.h> -#include <linux/serial_8250.h> - -#define PORT(base, int) \ -{ \ - .iobase = base, \ - .irq = int, \ - .uartclk = 1843200, \ - .iotype = UPIO_PORT, \ - .flags = UPF_BOOT_AUTOCONF | UPF_SKIP_TEST, \ - .regshift = 0, \ -} - -static struct plat_serial8250_port uart8250_data[] = { - PORT(0x3F8, 4), - PORT(0x2F8, 3), - PORT(0x3E8, 4), - PORT(0x2E8, 3), - { }, -}; - -static struct platform_device uart8250_device = { - .name = "serial8250", - .id = PLAT8250_DEV_PLATFORM, - .dev = { - .platform_data = uart8250_data, - }, -}; - -static int __init uart8250_init(void) -{ - return platform_device_register(&uart8250_device); -} - -module_init(uart8250_init); - -MODULE_AUTHOR("Ralf Baechle <ralf@linux-mips.org>"); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Generic 8250 UART probe driver"); diff --git a/arch/mips/kernel/Makefile b/arch/mips/kernel/Makefile index 423d871a946b..95a1e674fd67 100644 --- a/arch/mips/kernel/Makefile +++ b/arch/mips/kernel/Makefile @@ -1,117 +1,114 @@ +# SPDX-License-Identifier: GPL-2.0 # # Makefile for the Linux/MIPS kernel. # -extra-y := head.o vmlinux.lds +always-$(KBUILD_BUILTIN) := vmlinux.lds -obj-y += cpu-probe.o branch.o entry.o genex.o idle.o irq.o process.o \ - prom.o ptrace.o reset.o setup.o signal.o syscall.o \ - time.o topology.o traps.o unaligned.o watch.o vdso.o +obj-y += head.o branch.o cmpxchg.o elf.o entry.o genex.o idle.o irq.o \ + process.o prom.o ptrace.o reset.o setup.o signal.o \ + syscall.o time.o topology.o traps.o unaligned.o watch.o \ + vdso.o cacheinfo.o + +ifdef CONFIG_CPU_R3K_TLB +obj-y += cpu-r3k-probe.o +else +obj-y += cpu-probe.o +endif ifdef CONFIG_FUNCTION_TRACER -CFLAGS_REMOVE_ftrace.o = -pg -CFLAGS_REMOVE_early_printk.o = -pg -CFLAGS_REMOVE_perf_event.o = -pg -CFLAGS_REMOVE_perf_event_mipsxx.o = -pg +CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_early_printk.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_perf_event.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_perf_event_mipsxx.o = $(CC_FLAGS_FTRACE) endif obj-$(CONFIG_CEVT_BCM1480) += cevt-bcm1480.o obj-$(CONFIG_CEVT_R4K) += cevt-r4k.o -obj-$(CONFIG_MIPS_MT_SMTC) += cevt-smtc.o obj-$(CONFIG_CEVT_DS1287) += cevt-ds1287.o -obj-$(CONFIG_CEVT_GIC) += cevt-gic.o obj-$(CONFIG_CEVT_GT641XX) += cevt-gt641xx.o obj-$(CONFIG_CEVT_SB1250) += cevt-sb1250.o obj-$(CONFIG_CEVT_TXX9) += cevt-txx9.o obj-$(CONFIG_CSRC_BCM1480) += csrc-bcm1480.o -obj-$(CONFIG_CSRC_GIC) += csrc-gic.o obj-$(CONFIG_CSRC_IOASIC) += csrc-ioasic.o -obj-$(CONFIG_CSRC_POWERTV) += csrc-powertv.o obj-$(CONFIG_CSRC_R4K) += csrc-r4k.o obj-$(CONFIG_CSRC_SB1250) += csrc-sb1250.o obj-$(CONFIG_SYNC_R4K) += sync-r4k.o +obj-$(CONFIG_DEBUG_FS) += segment.o obj-$(CONFIG_STACKTRACE) += stacktrace.o -obj-$(CONFIG_MODULES) += mips_ksyms.o module.o -obj-$(CONFIG_MODULES_USE_ELF_RELA) += module-rela.o +obj-$(CONFIG_MODULES) += module.o +obj-$(CONFIG_FTRACE_SYSCALLS) += ftrace.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o -obj-$(CONFIG_CPU_R4K_FPU) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_R3000) += r2300_fpu.o r2300_switch.o -obj-$(CONFIG_CPU_R6000) += r6000_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_TX39XX) += r2300_fpu.o r2300_switch.o -obj-$(CONFIG_CPU_CAVIUM_OCTEON) += octeon_switch.o +sw-y := r4k_switch.o +sw-$(CONFIG_CPU_R3000) := r2300_switch.o +sw-$(CONFIG_CPU_CAVIUM_OCTEON) := octeon_switch.o +obj-y += $(sw-y) + +obj-$(CONFIG_MIPS_FP_SUPPORT) += fpu-probe.o +obj-$(CONFIG_CPU_R2300_FPU) += r2300_fpu.o +obj-$(CONFIG_CPU_R4K_FPU) += r4k_fpu.o obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_SMP_UP) += smp-up.o -obj-$(CONFIG_CPU_BMIPS) += smp-bmips.o bmips_vec.o +obj-$(CONFIG_CPU_BMIPS) += smp-bmips.o bmips_vec.o bmips_5xxx_init.o obj-$(CONFIG_MIPS_MT) += mips-mt.o obj-$(CONFIG_MIPS_MT_FPAFF) += mips-mt-fpaff.o -obj-$(CONFIG_MIPS_MT_SMTC) += smtc.o smtc-asm.o smtc-proc.o obj-$(CONFIG_MIPS_MT_SMP) += smp-mt.o -obj-$(CONFIG_MIPS_CMP) += smp-cmp.o -obj-$(CONFIG_CPU_MIPSR2) += spram.o +obj-$(CONFIG_MIPS_CPS) += smp-cps.o cps-vec.o +obj-$(CONFIG_MIPS_CPS_NS16550) += cps-vec-ns16550.o +obj-$(CONFIG_MIPS_SPRAM) += spram.o obj-$(CONFIG_MIPS_VPE_LOADER) += vpe.o +obj-$(CONFIG_MIPS_VPE_LOADER_MT) += vpe-mt.o obj-$(CONFIG_MIPS_VPE_APSP_API) += rtlx.o +obj-$(CONFIG_MIPS_VPE_APSP_API_MT) += rtlx-mt.o -obj-$(CONFIG_I8259) += i8259.o -obj-$(CONFIG_IRQ_CPU) += irq_cpu.o -obj-$(CONFIG_IRQ_CPU_RM7K) += irq-rm7000.o obj-$(CONFIG_MIPS_MSC) += irq-msc01.o obj-$(CONFIG_IRQ_TXX9) += irq_txx9.o obj-$(CONFIG_IRQ_GT641XX) += irq-gt641xx.o -obj-$(CONFIG_IRQ_GIC) += irq-gic.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_32BIT) += scall32-o32.o -obj-$(CONFIG_64BIT) += scall64-64.o +obj-$(CONFIG_64BIT) += scall64-n64.o obj-$(CONFIG_MIPS32_COMPAT) += linux32.o ptrace32.o signal32.o -obj-$(CONFIG_MIPS32_N32) += binfmt_elfn32.o scall64-n32.o signal_n32.o -obj-$(CONFIG_MIPS32_O32) += binfmt_elfo32.o scall64-o32.o +obj-$(CONFIG_MIPS32_N32) += scall64-n32.o signal_n32.o +obj-$(CONFIG_MIPS32_O32) += scall64-o32.o signal_o32.o obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_PROC_FS) += proc.o +obj-$(CONFIG_MAGIC_SYSRQ) += sysrq.o -obj-$(CONFIG_64BIT) += cpu-bugs64.o +obj-$(CONFIG_CPU_R4X00_BUGS64) += r4k-bugs64.o obj-$(CONFIG_I8253) += i8253.o obj-$(CONFIG_GPIO_TXX9) += gpio_txx9.o -obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o crash.o +obj-$(CONFIG_RELOCATABLE) += relocate.o + +obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o crash.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o +obj-$(CONFIG_EARLY_PRINTK_8250) += early_printk_8250.o obj-$(CONFIG_SPINLOCK_TEST) += spinlock_test.o -obj-$(CONFIG_MIPS_MACHINE) += mips_machine.o +obj-$(CONFIG_MIPSR2_TO_R6_EMULATOR) += mips-r2-to-r6-emul.o CFLAGS_cpu-bugs64.o = $(shell if $(CC) $(KBUILD_CFLAGS) -Wa,-mdaddi -c -o /dev/null -x c /dev/null >/dev/null 2>&1; then echo "-DHAVE_AS_SET_DADDI"; fi) -obj-$(CONFIG_HAVE_STD_PC_SERIAL_PORT) += 8250-platform.o - -obj-$(CONFIG_PERF_EVENTS) += perf_event.o +obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o obj-$(CONFIG_HW_PERF_EVENTS) += perf_event_mipsxx.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o +obj-$(CONFIG_UPROBES) += uprobes.o -# -# DSP ASE supported for MIPS32 or MIPS64 Release 2 cores only. It is not -# safe to unconditionnaly use the assembler -mdsp / -mdspr2 switches -# here because the compiler may use DSP ASE instructions (such as lwx) in -# code paths where we cannot check that the CPU we are running on supports it. -# Proper abstraction using HAVE_AS_DSP and macros is done in -# arch/mips/include/asm/mipsregs.h. -# -ifeq ($(CONFIG_CPU_MIPSR2), y) -CFLAGS_DSP = -DHAVE_AS_DSP - -CFLAGS_signal.o = $(CFLAGS_DSP) -CFLAGS_signal32.o = $(CFLAGS_DSP) -CFLAGS_process.o = $(CFLAGS_DSP) -CFLAGS_branch.o = $(CFLAGS_DSP) -CFLAGS_ptrace.o = $(CFLAGS_DSP) -endif +obj-$(CONFIG_MIPS_CM) += mips-cm.o +obj-$(CONFIG_MIPS_CPC) += mips-cpc.o + +obj-$(CONFIG_CPU_PM) += pm.o +obj-$(CONFIG_MIPS_CPS_PM) += pm-cps.o CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS) diff --git a/arch/mips/kernel/access-helper.h b/arch/mips/kernel/access-helper.h new file mode 100644 index 000000000000..590388031503 --- /dev/null +++ b/arch/mips/kernel/access-helper.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <linux/uaccess.h> + +static inline int __get_addr(unsigned long *a, unsigned long *p, bool user) +{ + return user ? get_user(*a, (unsigned long __user *)p) : + get_kernel_nofault(*a, p); +} + +static inline int __get_inst16(u16 *i, u16 *p, bool user) +{ + return user ? get_user(*i, (u16 __user *)p) : get_kernel_nofault(*i, p); +} + +static inline int __get_inst32(u32 *i, u32 *p, bool user) +{ + return user ? get_user(*i, (u32 __user *)p) : get_kernel_nofault(*i, p); +} diff --git a/arch/mips/kernel/asm-offsets.c b/arch/mips/kernel/asm-offsets.c index 0c2e853c3db4..5debd9a3854a 100644 --- a/arch/mips/kernel/asm-offsets.c +++ b/arch/mips/kernel/asm-offsets.c @@ -1,5 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 /* - * offset.c: Calculate pt_regs and task_struct offsets. + * asm-offsets.c: Calculate pt_regs and task_struct offsets. * * Copyright (C) 1996 David S. Miller * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003 Ralf Baechle @@ -8,20 +9,32 @@ * Kevin Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com * Copyright (C) 2000 MIPS Technologies, Inc. */ +#define COMPILE_OFFSETS + #include <linux/compat.h> #include <linux/types.h> #include <linux/sched.h> #include <linux/mm.h> #include <linux/kbuild.h> #include <linux/suspend.h> +#include <asm/cpu-info.h> +#include <asm/pm.h> #include <asm/ptrace.h> #include <asm/processor.h> +#include <asm/smp-cps.h> #include <linux/kvm_host.h> +void output_ptreg_defines(void); void output_ptreg_defines(void) { COMMENT("MIPS pt_regs offsets."); +#ifdef CONFIG_32BIT + OFFSET(PT_ARG4, pt_regs, args[4]); + OFFSET(PT_ARG5, pt_regs, args[5]); + OFFSET(PT_ARG6, pt_regs, args[6]); + OFFSET(PT_ARG7, pt_regs, args[7]); +#endif OFFSET(PT_R0, pt_regs, regs[0]); OFFSET(PT_R1, pt_regs, regs[1]); OFFSET(PT_R2, pt_regs, regs[2]); @@ -63,9 +76,6 @@ void output_ptreg_defines(void) OFFSET(PT_BVADDR, pt_regs, cp0_badvaddr); OFFSET(PT_STATUS, pt_regs, cp0_status); OFFSET(PT_CAUSE, pt_regs, cp0_cause); -#ifdef CONFIG_MIPS_MT_SMTC - OFFSET(PT_TCSTATUS, pt_regs, cp0_tcstatus); -#endif /* CONFIG_MIPS_MT_SMTC */ #ifdef CONFIG_CPU_CAVIUM_OCTEON OFFSET(PT_MPL, pt_regs, mpl); OFFSET(PT_MTP, pt_regs, mtp); @@ -74,38 +84,40 @@ void output_ptreg_defines(void) BLANK(); } +void output_task_defines(void); void output_task_defines(void) { COMMENT("MIPS task_struct offsets."); - OFFSET(TASK_STATE, task_struct, state); OFFSET(TASK_THREAD_INFO, task_struct, stack); OFFSET(TASK_FLAGS, task_struct, flags); OFFSET(TASK_MM, task_struct, mm); OFFSET(TASK_PID, task_struct, pid); -#if defined(CONFIG_CC_STACKPROTECTOR) +#if defined(CONFIG_STACKPROTECTOR) OFFSET(TASK_STACK_CANARY, task_struct, stack_canary); #endif DEFINE(TASK_STRUCT_SIZE, sizeof(struct task_struct)); BLANK(); } +void output_thread_info_defines(void); void output_thread_info_defines(void) { COMMENT("MIPS thread_info offsets."); OFFSET(TI_TASK, thread_info, task); - OFFSET(TI_EXEC_DOMAIN, thread_info, exec_domain); OFFSET(TI_FLAGS, thread_info, flags); OFFSET(TI_TP_VALUE, thread_info, tp_value); OFFSET(TI_CPU, thread_info, cpu); OFFSET(TI_PRE_COUNT, thread_info, preempt_count); - OFFSET(TI_ADDR_LIMIT, thread_info, addr_limit); - OFFSET(TI_RESTART_BLOCK, thread_info, restart_block); OFFSET(TI_REGS, thread_info, regs); + OFFSET(TI_SYSCALL, thread_info, syscall); DEFINE(_THREAD_SIZE, THREAD_SIZE); DEFINE(_THREAD_MASK, THREAD_MASK); + DEFINE(_IRQ_STACK_SIZE, IRQ_STACK_SIZE); + DEFINE(_IRQ_STACK_START, IRQ_STACK_START); BLANK(); } +void output_thread_defines(void); void output_thread_defines(void) { COMMENT("MIPS specific thread_struct offsets."); @@ -122,7 +134,6 @@ void output_thread_defines(void) OFFSET(THREAD_REG31, task_struct, thread.reg31); OFFSET(THREAD_STATUS, task_struct, thread.cp0_status); - OFFSET(THREAD_FPU, task_struct, thread.fpu); OFFSET(THREAD_BVADDR, task_struct, \ thread.cp0_badvaddr); @@ -130,11 +141,16 @@ void output_thread_defines(void) thread.cp0_baduaddr); OFFSET(THREAD_ECODE, task_struct, \ thread.error_code); + OFFSET(THREAD_TRAPNO, task_struct, thread.trap_nr); BLANK(); } +#ifdef CONFIG_MIPS_FP_SUPPORT +void output_thread_fpu_defines(void); void output_thread_fpu_defines(void) { + OFFSET(THREAD_FPU, task_struct, thread.fpu); + OFFSET(THREAD_FPR0, task_struct, thread.fpu.fpr[0]); OFFSET(THREAD_FPR1, task_struct, thread.fpu.fpr[1]); OFFSET(THREAD_FPR2, task_struct, thread.fpu.fpr[2]); @@ -169,9 +185,12 @@ void output_thread_fpu_defines(void) OFFSET(THREAD_FPR31, task_struct, thread.fpu.fpr[31]); OFFSET(THREAD_FCR31, task_struct, thread.fpu.fcr31); + OFFSET(THREAD_MSA_CSR, task_struct, thread.fpu.msacsr); BLANK(); } +#endif +void output_mm_defines(void); void output_mm_defines(void) { COMMENT("Size of struct page"); @@ -192,11 +211,6 @@ void output_mm_defines(void) #endif DEFINE(_PTE_T_LOG2, PTE_T_LOG2); BLANK(); - DEFINE(_PGD_ORDER, PGD_ORDER); -#ifndef __PAGETABLE_PMD_FOLDED - DEFINE(_PMD_ORDER, PMD_ORDER); -#endif - DEFINE(_PTE_ORDER, PTE_ORDER); BLANK(); DEFINE(_PMD_SHIFT, PMD_SHIFT); DEFINE(_PGDIR_SHIFT, PGDIR_SHIFT); @@ -211,6 +225,7 @@ void output_mm_defines(void) } #ifdef CONFIG_32BIT +void output_sc_defines(void); void output_sc_defines(void) { COMMENT("Linux sigcontext offsets."); @@ -233,6 +248,7 @@ void output_sc_defines(void) #endif #ifdef CONFIG_64BIT +void output_sc_defines(void); void output_sc_defines(void) { COMMENT("Linux sigcontext offsets."); @@ -246,17 +262,7 @@ void output_sc_defines(void) } #endif -#ifdef CONFIG_MIPS32_COMPAT -void output_sc32_defines(void) -{ - COMMENT("Linux 32-bit sigcontext offsets."); - OFFSET(SC32_FPREGS, sigcontext32, sc_fpregs); - OFFSET(SC32_FPC_CSR, sigcontext32, sc_fpc_csr); - OFFSET(SC32_FPC_EIR, sigcontext32, sc_fpc_eir); - BLANK(); -} -#endif - +void output_signal_defined(void); void output_signal_defined(void) { COMMENT("Linux signal numbers."); @@ -296,6 +302,7 @@ void output_signal_defined(void) } #ifdef CONFIG_CPU_CAVIUM_OCTEON +void output_octeon_cop2_state_defines(void); void output_octeon_cop2_state_defines(void) { COMMENT("Octeon specific octeon_cop2_state offsets."); @@ -316,13 +323,18 @@ void output_octeon_cop2_state_defines(void) OFFSET(OCTEON_CP2_GFM_RESULT, octeon_cop2_state, cop2_gfm_result); OFFSET(OCTEON_CP2_HSH_DATW, octeon_cop2_state, cop2_hsh_datw); OFFSET(OCTEON_CP2_HSH_IVW, octeon_cop2_state, cop2_hsh_ivw); + OFFSET(OCTEON_CP2_SHA3, octeon_cop2_state, cop2_sha3); OFFSET(THREAD_CP2, task_struct, thread.cp2); +#if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \ + CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0 OFFSET(THREAD_CVMSEG, task_struct, thread.cvmseg.cvmseg); +#endif BLANK(); } #endif #ifdef CONFIG_HIBERNATION +void output_pbe_defines(void); void output_pbe_defines(void) { COMMENT(" Linux struct pbe offsets. "); @@ -334,66 +346,82 @@ void output_pbe_defines(void) } #endif +#ifdef CONFIG_CPU_PM +void output_pm_defines(void); +void output_pm_defines(void) +{ + COMMENT(" PM offsets. "); +#ifdef CONFIG_EVA + OFFSET(SSS_SEGCTL0, mips_static_suspend_state, segctl[0]); + OFFSET(SSS_SEGCTL1, mips_static_suspend_state, segctl[1]); + OFFSET(SSS_SEGCTL2, mips_static_suspend_state, segctl[2]); +#endif + OFFSET(SSS_SP, mips_static_suspend_state, sp); + BLANK(); +} +#endif + +#ifdef CONFIG_MIPS_FP_SUPPORT +void output_kvm_defines(void); void output_kvm_defines(void) { - COMMENT(" KVM/MIPS Specfic offsets. "); - DEFINE(VCPU_ARCH_SIZE, sizeof(struct kvm_vcpu_arch)); - OFFSET(VCPU_RUN, kvm_vcpu, run); - OFFSET(VCPU_HOST_ARCH, kvm_vcpu, arch); + COMMENT(" KVM/MIPS Specific offsets. "); - OFFSET(VCPU_HOST_EBASE, kvm_vcpu_arch, host_ebase); - OFFSET(VCPU_GUEST_EBASE, kvm_vcpu_arch, guest_ebase); + OFFSET(VCPU_FPR0, kvm_vcpu_arch, fpu.fpr[0]); + OFFSET(VCPU_FPR1, kvm_vcpu_arch, fpu.fpr[1]); + OFFSET(VCPU_FPR2, kvm_vcpu_arch, fpu.fpr[2]); + OFFSET(VCPU_FPR3, kvm_vcpu_arch, fpu.fpr[3]); + OFFSET(VCPU_FPR4, kvm_vcpu_arch, fpu.fpr[4]); + OFFSET(VCPU_FPR5, kvm_vcpu_arch, fpu.fpr[5]); + OFFSET(VCPU_FPR6, kvm_vcpu_arch, fpu.fpr[6]); + OFFSET(VCPU_FPR7, kvm_vcpu_arch, fpu.fpr[7]); + OFFSET(VCPU_FPR8, kvm_vcpu_arch, fpu.fpr[8]); + OFFSET(VCPU_FPR9, kvm_vcpu_arch, fpu.fpr[9]); + OFFSET(VCPU_FPR10, kvm_vcpu_arch, fpu.fpr[10]); + OFFSET(VCPU_FPR11, kvm_vcpu_arch, fpu.fpr[11]); + OFFSET(VCPU_FPR12, kvm_vcpu_arch, fpu.fpr[12]); + OFFSET(VCPU_FPR13, kvm_vcpu_arch, fpu.fpr[13]); + OFFSET(VCPU_FPR14, kvm_vcpu_arch, fpu.fpr[14]); + OFFSET(VCPU_FPR15, kvm_vcpu_arch, fpu.fpr[15]); + OFFSET(VCPU_FPR16, kvm_vcpu_arch, fpu.fpr[16]); + OFFSET(VCPU_FPR17, kvm_vcpu_arch, fpu.fpr[17]); + OFFSET(VCPU_FPR18, kvm_vcpu_arch, fpu.fpr[18]); + OFFSET(VCPU_FPR19, kvm_vcpu_arch, fpu.fpr[19]); + OFFSET(VCPU_FPR20, kvm_vcpu_arch, fpu.fpr[20]); + OFFSET(VCPU_FPR21, kvm_vcpu_arch, fpu.fpr[21]); + OFFSET(VCPU_FPR22, kvm_vcpu_arch, fpu.fpr[22]); + OFFSET(VCPU_FPR23, kvm_vcpu_arch, fpu.fpr[23]); + OFFSET(VCPU_FPR24, kvm_vcpu_arch, fpu.fpr[24]); + OFFSET(VCPU_FPR25, kvm_vcpu_arch, fpu.fpr[25]); + OFFSET(VCPU_FPR26, kvm_vcpu_arch, fpu.fpr[26]); + OFFSET(VCPU_FPR27, kvm_vcpu_arch, fpu.fpr[27]); + OFFSET(VCPU_FPR28, kvm_vcpu_arch, fpu.fpr[28]); + OFFSET(VCPU_FPR29, kvm_vcpu_arch, fpu.fpr[29]); + OFFSET(VCPU_FPR30, kvm_vcpu_arch, fpu.fpr[30]); + OFFSET(VCPU_FPR31, kvm_vcpu_arch, fpu.fpr[31]); - OFFSET(VCPU_HOST_STACK, kvm_vcpu_arch, host_stack); - OFFSET(VCPU_HOST_GP, kvm_vcpu_arch, host_gp); + OFFSET(VCPU_FCR31, kvm_vcpu_arch, fpu.fcr31); + OFFSET(VCPU_MSA_CSR, kvm_vcpu_arch, fpu.msacsr); + BLANK(); +} +#endif - OFFSET(VCPU_HOST_CP0_BADVADDR, kvm_vcpu_arch, host_cp0_badvaddr); - OFFSET(VCPU_HOST_CP0_CAUSE, kvm_vcpu_arch, host_cp0_cause); - OFFSET(VCPU_HOST_EPC, kvm_vcpu_arch, host_cp0_epc); - OFFSET(VCPU_HOST_ENTRYHI, kvm_vcpu_arch, host_cp0_entryhi); +#ifdef CONFIG_MIPS_CPS +void output_cps_defines(void); +void output_cps_defines(void) +{ + COMMENT(" MIPS CPS offsets. "); - OFFSET(VCPU_GUEST_INST, kvm_vcpu_arch, guest_inst); + OFFSET(CLUSTERBOOTCFG_CORECONFIG, cluster_boot_config, core_config); + DEFINE(CLUSTERBOOTCFG_SIZE, sizeof(struct cluster_boot_config)); - OFFSET(VCPU_R0, kvm_vcpu_arch, gprs[0]); - OFFSET(VCPU_R1, kvm_vcpu_arch, gprs[1]); - OFFSET(VCPU_R2, kvm_vcpu_arch, gprs[2]); - OFFSET(VCPU_R3, kvm_vcpu_arch, gprs[3]); - OFFSET(VCPU_R4, kvm_vcpu_arch, gprs[4]); - OFFSET(VCPU_R5, kvm_vcpu_arch, gprs[5]); - OFFSET(VCPU_R6, kvm_vcpu_arch, gprs[6]); - OFFSET(VCPU_R7, kvm_vcpu_arch, gprs[7]); - OFFSET(VCPU_R8, kvm_vcpu_arch, gprs[8]); - OFFSET(VCPU_R9, kvm_vcpu_arch, gprs[9]); - OFFSET(VCPU_R10, kvm_vcpu_arch, gprs[10]); - OFFSET(VCPU_R11, kvm_vcpu_arch, gprs[11]); - OFFSET(VCPU_R12, kvm_vcpu_arch, gprs[12]); - OFFSET(VCPU_R13, kvm_vcpu_arch, gprs[13]); - OFFSET(VCPU_R14, kvm_vcpu_arch, gprs[14]); - OFFSET(VCPU_R15, kvm_vcpu_arch, gprs[15]); - OFFSET(VCPU_R16, kvm_vcpu_arch, gprs[16]); - OFFSET(VCPU_R17, kvm_vcpu_arch, gprs[17]); - OFFSET(VCPU_R18, kvm_vcpu_arch, gprs[18]); - OFFSET(VCPU_R19, kvm_vcpu_arch, gprs[19]); - OFFSET(VCPU_R20, kvm_vcpu_arch, gprs[20]); - OFFSET(VCPU_R21, kvm_vcpu_arch, gprs[21]); - OFFSET(VCPU_R22, kvm_vcpu_arch, gprs[22]); - OFFSET(VCPU_R23, kvm_vcpu_arch, gprs[23]); - OFFSET(VCPU_R24, kvm_vcpu_arch, gprs[24]); - OFFSET(VCPU_R25, kvm_vcpu_arch, gprs[25]); - OFFSET(VCPU_R26, kvm_vcpu_arch, gprs[26]); - OFFSET(VCPU_R27, kvm_vcpu_arch, gprs[27]); - OFFSET(VCPU_R28, kvm_vcpu_arch, gprs[28]); - OFFSET(VCPU_R29, kvm_vcpu_arch, gprs[29]); - OFFSET(VCPU_R30, kvm_vcpu_arch, gprs[30]); - OFFSET(VCPU_R31, kvm_vcpu_arch, gprs[31]); - OFFSET(VCPU_LO, kvm_vcpu_arch, lo); - OFFSET(VCPU_HI, kvm_vcpu_arch, hi); - OFFSET(VCPU_PC, kvm_vcpu_arch, pc); - OFFSET(VCPU_COP0, kvm_vcpu_arch, cop0); - OFFSET(VCPU_GUEST_KERNEL_ASID, kvm_vcpu_arch, guest_kernel_asid); - OFFSET(VCPU_GUEST_USER_ASID, kvm_vcpu_arch, guest_user_asid); + OFFSET(COREBOOTCFG_VPEMASK, core_boot_config, vpe_mask); + OFFSET(COREBOOTCFG_VPECONFIG, core_boot_config, vpe_config); + DEFINE(COREBOOTCFG_SIZE, sizeof(struct core_boot_config)); - OFFSET(COP0_TLB_HI, mips_coproc, reg[MIPS_CP0_TLB_HI][0]); - OFFSET(COP0_STATUS, mips_coproc, reg[MIPS_CP0_STATUS][0]); - BLANK(); + OFFSET(VPEBOOTCFG_PC, vpe_boot_config, pc); + OFFSET(VPEBOOTCFG_SP, vpe_boot_config, sp); + OFFSET(VPEBOOTCFG_GP, vpe_boot_config, gp); + DEFINE(VPEBOOTCFG_SIZE, sizeof(struct vpe_boot_config)); } +#endif diff --git a/arch/mips/kernel/binfmt_elfn32.c b/arch/mips/kernel/binfmt_elfn32.c deleted file mode 100644 index 1188e00bb120..000000000000 --- a/arch/mips/kernel/binfmt_elfn32.c +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Support for n32 Linux/MIPS ELF binaries. - * - * Copyright (C) 1999, 2001 Ralf Baechle - * Copyright (C) 1999, 2001 Silicon Graphics, Inc. - * - * Heavily inspired by the 32-bit Sparc compat code which is - * Copyright (C) 1995, 1996, 1997, 1998 David S. Miller (davem@redhat.com) - * Copyright (C) 1995, 1996, 1997, 1998 Jakub Jelinek (jj@ultra.linux.cz) - */ - -#define ELF_ARCH EM_MIPS -#define ELF_CLASS ELFCLASS32 -#ifdef __MIPSEB__ -#define ELF_DATA ELFDATA2MSB; -#else /* __MIPSEL__ */ -#define ELF_DATA ELFDATA2LSB; -#endif - -/* ELF register definitions */ -#define ELF_NGREG 45 -#define ELF_NFPREG 33 - -typedef unsigned long elf_greg_t; -typedef elf_greg_t elf_gregset_t[ELF_NGREG]; - -typedef double elf_fpreg_t; -typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; - -/* - * This is used to ensure we don't load something for the wrong architecture. - */ -#define elf_check_arch(hdr) \ -({ \ - int __res = 1; \ - struct elfhdr *__h = (hdr); \ - \ - if (__h->e_machine != EM_MIPS) \ - __res = 0; \ - if (__h->e_ident[EI_CLASS] != ELFCLASS32) \ - __res = 0; \ - if (((__h->e_flags & EF_MIPS_ABI2) == 0) || \ - ((__h->e_flags & EF_MIPS_ABI) != 0)) \ - __res = 0; \ - \ - __res; \ -}) - -#define TASK32_SIZE 0x7fff8000UL -#undef ELF_ET_DYN_BASE -#define ELF_ET_DYN_BASE (TASK32_SIZE / 3 * 2) - -#include <asm/processor.h> -#include <linux/module.h> -#include <linux/elfcore.h> -#include <linux/compat.h> -#include <linux/math64.h> - -#define elf_prstatus elf_prstatus32 -struct elf_prstatus32 -{ - struct elf_siginfo pr_info; /* Info associated with signal */ - short pr_cursig; /* Current signal */ - unsigned int pr_sigpend; /* Set of pending signals */ - unsigned int pr_sighold; /* Set of held signals */ - pid_t pr_pid; - pid_t pr_ppid; - pid_t pr_pgrp; - pid_t pr_sid; - struct compat_timeval pr_utime; /* User time */ - struct compat_timeval pr_stime; /* System time */ - struct compat_timeval pr_cutime;/* Cumulative user time */ - struct compat_timeval pr_cstime;/* Cumulative system time */ - elf_gregset_t pr_reg; /* GP registers */ - int pr_fpvalid; /* True if math co-processor being used. */ -}; - -#define elf_prpsinfo elf_prpsinfo32 -struct elf_prpsinfo32 -{ - char pr_state; /* numeric process state */ - char pr_sname; /* char for pr_state */ - char pr_zomb; /* zombie */ - char pr_nice; /* nice val */ - unsigned int pr_flag; /* flags */ - __kernel_uid_t pr_uid; - __kernel_gid_t pr_gid; - pid_t pr_pid, pr_ppid, pr_pgrp, pr_sid; - /* Lots missing */ - char pr_fname[16]; /* filename of executable */ - char pr_psargs[ELF_PRARGSZ]; /* initial part of arg list */ -}; - -#define elf_caddr_t u32 -#define init_elf_binfmt init_elfn32_binfmt - -#define jiffies_to_timeval jiffies_to_compat_timeval -static __inline__ void -jiffies_to_compat_timeval(unsigned long jiffies, struct compat_timeval *value) -{ - /* - * Convert jiffies to nanoseconds and separate with - * one divide. - */ - u64 nsec = (u64)jiffies * TICK_NSEC; - u32 rem; - value->tv_sec = div_u64_rem(nsec, NSEC_PER_SEC, &rem); - value->tv_usec = rem / NSEC_PER_USEC; -} - -#define ELF_CORE_EFLAGS EF_MIPS_ABI2 - -MODULE_DESCRIPTION("Binary format loader for compatibility with n32 Linux/MIPS binaries"); -MODULE_AUTHOR("Ralf Baechle (ralf@linux-mips.org)"); - -#undef MODULE_DESCRIPTION -#undef MODULE_AUTHOR - -#undef TASK_SIZE -#define TASK_SIZE TASK_SIZE32 - -#undef cputime_to_timeval -#define cputime_to_timeval cputime_to_compat_timeval -static __inline__ void -cputime_to_compat_timeval(const cputime_t cputime, struct compat_timeval *value) -{ - unsigned long jiffies = cputime_to_jiffies(cputime); - - value->tv_usec = (jiffies % HZ) * (1000000L / HZ); - value->tv_sec = jiffies / HZ; -} - -#include "../../../fs/binfmt_elf.c" diff --git a/arch/mips/kernel/binfmt_elfo32.c b/arch/mips/kernel/binfmt_elfo32.c deleted file mode 100644 index 202e581e6096..000000000000 --- a/arch/mips/kernel/binfmt_elfo32.c +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Support for o32 Linux/MIPS ELF binaries. - * - * Copyright (C) 1999, 2001 Ralf Baechle - * Copyright (C) 1999, 2001 Silicon Graphics, Inc. - * - * Heavily inspired by the 32-bit Sparc compat code which is - * Copyright (C) 1995, 1996, 1997, 1998 David S. Miller (davem@redhat.com) - * Copyright (C) 1995, 1996, 1997, 1998 Jakub Jelinek (jj@ultra.linux.cz) - */ - -#define ELF_ARCH EM_MIPS -#define ELF_CLASS ELFCLASS32 -#ifdef __MIPSEB__ -#define ELF_DATA ELFDATA2MSB; -#else /* __MIPSEL__ */ -#define ELF_DATA ELFDATA2LSB; -#endif - -/* ELF register definitions */ -#define ELF_NGREG 45 -#define ELF_NFPREG 33 - -typedef unsigned int elf_greg_t; -typedef elf_greg_t elf_gregset_t[ELF_NGREG]; - -typedef double elf_fpreg_t; -typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; - -/* - * This is used to ensure we don't load something for the wrong architecture. - */ -#define elf_check_arch(hdr) \ -({ \ - int __res = 1; \ - struct elfhdr *__h = (hdr); \ - \ - if (__h->e_machine != EM_MIPS) \ - __res = 0; \ - if (__h->e_ident[EI_CLASS] != ELFCLASS32) \ - __res = 0; \ - if ((__h->e_flags & EF_MIPS_ABI2) != 0) \ - __res = 0; \ - if (((__h->e_flags & EF_MIPS_ABI) != 0) && \ - ((__h->e_flags & EF_MIPS_ABI) != EF_MIPS_ABI_O32)) \ - __res = 0; \ - \ - __res; \ -}) - -#ifdef CONFIG_KVM_GUEST -#define TASK32_SIZE 0x3fff8000UL -#else -#define TASK32_SIZE 0x7fff8000UL -#endif -#undef ELF_ET_DYN_BASE -#define ELF_ET_DYN_BASE (TASK32_SIZE / 3 * 2) - -#include <asm/processor.h> - -/* - * When this file is selected, we are definitely running a 64bit kernel. - * So using the right regs define in asm/reg.h - */ -#define WANT_COMPAT_REG_H - -/* These MUST be defined before elf.h gets included */ -extern void elf32_core_copy_regs(elf_gregset_t grp, struct pt_regs *regs); -#define ELF_CORE_COPY_REGS(_dest, _regs) elf32_core_copy_regs(_dest, _regs); -#define ELF_CORE_COPY_TASK_REGS(_tsk, _dest) \ -({ \ - int __res = 1; \ - elf32_core_copy_regs(*(_dest), task_pt_regs(_tsk)); \ - __res; \ -}) - -#include <linux/module.h> -#include <linux/elfcore.h> -#include <linux/compat.h> -#include <linux/math64.h> - -#define elf_prstatus elf_prstatus32 -struct elf_prstatus32 -{ - struct elf_siginfo pr_info; /* Info associated with signal */ - short pr_cursig; /* Current signal */ - unsigned int pr_sigpend; /* Set of pending signals */ - unsigned int pr_sighold; /* Set of held signals */ - pid_t pr_pid; - pid_t pr_ppid; - pid_t pr_pgrp; - pid_t pr_sid; - struct compat_timeval pr_utime; /* User time */ - struct compat_timeval pr_stime; /* System time */ - struct compat_timeval pr_cutime;/* Cumulative user time */ - struct compat_timeval pr_cstime;/* Cumulative system time */ - elf_gregset_t pr_reg; /* GP registers */ - int pr_fpvalid; /* True if math co-processor being used. */ -}; - -#define elf_prpsinfo elf_prpsinfo32 -struct elf_prpsinfo32 -{ - char pr_state; /* numeric process state */ - char pr_sname; /* char for pr_state */ - char pr_zomb; /* zombie */ - char pr_nice; /* nice val */ - unsigned int pr_flag; /* flags */ - __kernel_uid_t pr_uid; - __kernel_gid_t pr_gid; - pid_t pr_pid, pr_ppid, pr_pgrp, pr_sid; - /* Lots missing */ - char pr_fname[16]; /* filename of executable */ - char pr_psargs[ELF_PRARGSZ]; /* initial part of arg list */ -}; - -#define elf_caddr_t u32 -#define init_elf_binfmt init_elf32_binfmt - -#define jiffies_to_timeval jiffies_to_compat_timeval -static inline void -jiffies_to_compat_timeval(unsigned long jiffies, struct compat_timeval *value) -{ - /* - * Convert jiffies to nanoseconds and separate with - * one divide. - */ - u64 nsec = (u64)jiffies * TICK_NSEC; - u32 rem; - value->tv_sec = div_u64_rem(nsec, NSEC_PER_SEC, &rem); - value->tv_usec = rem / NSEC_PER_USEC; -} - -void elf32_core_copy_regs(elf_gregset_t grp, struct pt_regs *regs) -{ - int i; - - for (i = 0; i < EF_R0; i++) - grp[i] = 0; - grp[EF_R0] = 0; - for (i = 1; i <= 31; i++) - grp[EF_R0 + i] = (elf_greg_t) regs->regs[i]; - grp[EF_R26] = 0; - grp[EF_R27] = 0; - grp[EF_LO] = (elf_greg_t) regs->lo; - grp[EF_HI] = (elf_greg_t) regs->hi; - grp[EF_CP0_EPC] = (elf_greg_t) regs->cp0_epc; - grp[EF_CP0_BADVADDR] = (elf_greg_t) regs->cp0_badvaddr; - grp[EF_CP0_STATUS] = (elf_greg_t) regs->cp0_status; - grp[EF_CP0_CAUSE] = (elf_greg_t) regs->cp0_cause; -#ifdef EF_UNUSED0 - grp[EF_UNUSED0] = 0; -#endif -} - -MODULE_DESCRIPTION("Binary format loader for compatibility with o32 Linux/MIPS binaries"); -MODULE_AUTHOR("Ralf Baechle (ralf@linux-mips.org)"); - -#undef MODULE_DESCRIPTION -#undef MODULE_AUTHOR - -#undef TASK_SIZE -#define TASK_SIZE TASK_SIZE32 - -#undef cputime_to_timeval -#define cputime_to_timeval cputime_to_compat_timeval -static __inline__ void -cputime_to_compat_timeval(const cputime_t cputime, struct compat_timeval *value) -{ - unsigned long jiffies = cputime_to_jiffies(cputime); - - value->tv_usec = (jiffies % HZ) * (1000000L / HZ); - value->tv_sec = jiffies / HZ; -} - -#include "../../../fs/binfmt_elf.c" diff --git a/arch/mips/kernel/bmips_5xxx_init.S b/arch/mips/kernel/bmips_5xxx_init.S new file mode 100644 index 000000000000..9e422d186a17 --- /dev/null +++ b/arch/mips/kernel/bmips_5xxx_init.S @@ -0,0 +1,747 @@ + +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2011-2012 by Broadcom Corporation + * + * Init for bmips 5000. + * Used to init second core in dual core 5000's. + */ + +#include <linux/init.h> + +#include <asm/asm.h> +#include <asm/asmmacro.h> +#include <asm/cacheops.h> +#include <asm/regdef.h> +#include <asm/mipsregs.h> +#include <asm/stackframe.h> +#include <asm/addrspace.h> +#include <asm/hazards.h> +#include <asm/bmips.h> + +#ifdef CONFIG_CPU_BMIPS5000 + + +#define cacheop(kva, size, linesize, op) \ + .set noreorder ; \ + addu t1, kva, size ; \ + subu t2, linesize, 1 ; \ + not t2 ; \ + and t0, kva, t2 ; \ + addiu t1, t1, -1 ; \ + and t1, t2 ; \ +9: cache op, 0(t0) ; \ + bne t0, t1, 9b ; \ + addu t0, linesize ; \ + .set reorder ; + + + +#define IS_SHIFT 22 +#define IL_SHIFT 19 +#define IA_SHIFT 16 +#define DS_SHIFT 13 +#define DL_SHIFT 10 +#define DA_SHIFT 7 +#define IS_MASK 7 +#define IL_MASK 7 +#define IA_MASK 7 +#define DS_MASK 7 +#define DL_MASK 7 +#define DA_MASK 7 +#define ICE_MASK 0x80000000 +#define DCE_MASK 0x40000000 + +#define CP0_BRCM_CONFIG0 $22, 0 +#define CP0_BRCM_MODE $22, 1 +#define CP0_CONFIG_K0_MASK 7 + +#define CP0_ICACHE_TAG_LO $28 +#define CP0_ICACHE_DATA_LO $28, 1 +#define CP0_DCACHE_TAG_LO $28, 2 +#define CP0_D_SEC_CACHE_DATA_LO $28, 3 +#define CP0_ICACHE_TAG_HI $29 +#define CP0_ICACHE_DATA_HI $29, 1 +#define CP0_DCACHE_TAG_HI $29, 2 + +#define CP0_BRCM_MODE_Luc_MASK (1 << 11) +#define CP0_BRCM_CONFIG0_CWF_MASK (1 << 20) +#define CP0_BRCM_CONFIG0_TSE_MASK (1 << 19) +#define CP0_BRCM_MODE_SET_MASK (1 << 7) +#define CP0_BRCM_MODE_ClkRATIO_MASK (7 << 4) +#define CP0_BRCM_MODE_BrPRED_MASK (3 << 24) +#define CP0_BRCM_MODE_BrPRED_SHIFT 24 +#define CP0_BRCM_MODE_BrHIST_MASK (0x1f << 20) +#define CP0_BRCM_MODE_BrHIST_SHIFT 20 + +/* ZSC L2 Cache Register Access Register Definitions */ +#define BRCM_ZSC_ALL_REGS_SELECT 0x7 << 24 + +#define BRCM_ZSC_CONFIG_REG 0 << 3 +#define BRCM_ZSC_REQ_BUFFER_REG 2 << 3 +#define BRCM_ZSC_RBUS_ADDR_MAPPING_REG0 4 << 3 +#define BRCM_ZSC_RBUS_ADDR_MAPPING_REG1 6 << 3 +#define BRCM_ZSC_RBUS_ADDR_MAPPING_REG2 8 << 3 + +#define BRCM_ZSC_SCB0_ADDR_MAPPING_REG0 0xa << 3 +#define BRCM_ZSC_SCB0_ADDR_MAPPING_REG1 0xc << 3 + +#define BRCM_ZSC_SCB1_ADDR_MAPPING_REG0 0xe << 3 +#define BRCM_ZSC_SCB1_ADDR_MAPPING_REG1 0x10 << 3 + +#define BRCM_ZSC_CONFIG_LMB1En 1 << (15) +#define BRCM_ZSC_CONFIG_LMB0En 1 << (14) + +/* branch predition values */ + +#define BRCM_BrPRED_ALL_TAKEN (0x0) +#define BRCM_BrPRED_ALL_NOT_TAKEN (0x1) +#define BRCM_BrPRED_BHT_ENABLE (0x2) +#define BRCM_BrPRED_PREDICT_BACKWARD (0x3) + + + +.align 2 +/* + * Function: size_i_cache + * Arguments: None + * Returns: v0 = i cache size, v1 = I cache line size + * Description: compute the I-cache size and I-cache line size + * Trashes: v0, v1, a0, t0 + * + * pseudo code: + * + */ + +LEAF(size_i_cache) + .set noreorder + + mfc0 a0, CP0_CONFIG, 1 + move t0, a0 + + /* + * Determine sets per way: IS + * + * This field contains the number of sets (i.e., indices) per way of + * the instruction cache: + * i) 0x0: 64, ii) 0x1: 128, iii) 0x2: 256, iv) 0x3: 512, v) 0x4: 1k + * vi) 0x5 - 0x7: Reserved. + */ + + srl a0, a0, IS_SHIFT + and a0, a0, IS_MASK + + /* sets per way = (64<<IS) */ + + li v0, 0x40 + sllv v0, v0, a0 + + /* + * Determine line size + * + * This field contains the line size of the instruction cache: + * i) 0x0: No I-cache present, i) 0x3: 16 bytes, ii) 0x4: 32 bytes, iii) + * 0x5: 64 bytes, iv) the rest: Reserved. + */ + + move a0, t0 + + srl a0, a0, IL_SHIFT + and a0, a0, IL_MASK + + beqz a0, no_i_cache + nop + + /* line size = 2 ^ (IL+1) */ + + addi a0, a0, 1 + li v1, 1 + sll v1, v1, a0 + + /* v0 now have sets per way, multiply it by line size now + * that will give the set size + */ + + sll v0, v0, a0 + + /* + * Determine set associativity + * + * This field contains the set associativity of the instruction cache. + * i) 0x0: Direct mapped, ii) 0x1: 2-way, iii) 0x2: 3-way, iv) 0x3: + * 4-way, v) 0x4 - 0x7: Reserved. + */ + + move a0, t0 + + srl a0, a0, IA_SHIFT + and a0, a0, IA_MASK + addi a0, a0, 0x1 + + /* v0 has the set size, multiply it by + * set associativiy, to get the cache size + */ + + multu v0, a0 /*multu is interlocked, so no need to insert nops */ + mflo v0 + b 1f + nop + +no_i_cache: + move v0, zero + move v1, zero +1: + jr ra + nop + .set reorder + +END(size_i_cache) + +/* + * Function: size_d_cache + * Arguments: None + * Returns: v0 = d cache size, v1 = d cache line size + * Description: compute the D-cache size and D-cache line size. + * Trashes: v0, v1, a0, t0 + * + */ + +LEAF(size_d_cache) + .set noreorder + + mfc0 a0, CP0_CONFIG, 1 + move t0, a0 + + /* + * Determine sets per way: IS + * + * This field contains the number of sets (i.e., indices) per way of + * the instruction cache: + * i) 0x0: 64, ii) 0x1: 128, iii) 0x2: 256, iv) 0x3: 512, v) 0x4: 1k + * vi) 0x5 - 0x7: Reserved. + */ + + srl a0, a0, DS_SHIFT + and a0, a0, DS_MASK + + /* sets per way = (64<<IS) */ + + li v0, 0x40 + sllv v0, v0, a0 + + /* + * Determine line size + * + * This field contains the line size of the instruction cache: + * i) 0x0: No I-cache present, i) 0x3: 16 bytes, ii) 0x4: 32 bytes, iii) + * 0x5: 64 bytes, iv) the rest: Reserved. + */ + move a0, t0 + + srl a0, a0, DL_SHIFT + and a0, a0, DL_MASK + + beqz a0, no_d_cache + nop + + /* line size = 2 ^ (IL+1) */ + + addi a0, a0, 1 + li v1, 1 + sll v1, v1, a0 + + /* v0 now have sets per way, multiply it by line size now + * that will give the set size + */ + + sll v0, v0, a0 + + /* determine set associativity + * + * This field contains the set associativity of the instruction cache. + * i) 0x0: Direct mapped, ii) 0x1: 2-way, iii) 0x2: 3-way, iv) 0x3: + * 4-way, v) 0x4 - 0x7: Reserved. + */ + + move a0, t0 + + srl a0, a0, DA_SHIFT + and a0, a0, DA_MASK + addi a0, a0, 0x1 + + /* v0 has the set size, multiply it by + * set associativiy, to get the cache size + */ + + multu v0, a0 /*multu is interlocked, so no need to insert nops */ + mflo v0 + + b 1f + nop + +no_d_cache: + move v0, zero + move v1, zero +1: + jr ra + nop + .set reorder + +END(size_d_cache) + + +/* + * Function: enable_ID + * Arguments: None + * Returns: None + * Description: Enable I and D caches, initialize I and D-caches, also set + * hardware delay for d-cache (TP0). + * Trashes: t0 + * + */ + .global enable_ID + .ent enable_ID + .set noreorder +enable_ID: + mfc0 t0, CP0_BRCM_CONFIG0 + or t0, t0, (ICE_MASK | DCE_MASK) + mtc0 t0, CP0_BRCM_CONFIG0 + jr ra + nop + + .end enable_ID + .set reorder + + +/* + * Function: l1_init + * Arguments: None + * Returns: None + * Description: Enable I and D caches, and initialize I and D-caches + * Trashes: a0, v0, v1, t0, t1, t2, t8 + * + */ + .globl l1_init + .ent l1_init + .set noreorder +l1_init: + + /* save return address */ + move t8, ra + + + /* initialize I and D cache Data and Tag registers. */ + mtc0 zero, CP0_ICACHE_TAG_LO + mtc0 zero, CP0_ICACHE_TAG_HI + mtc0 zero, CP0_ICACHE_DATA_LO + mtc0 zero, CP0_ICACHE_DATA_HI + mtc0 zero, CP0_DCACHE_TAG_LO + mtc0 zero, CP0_DCACHE_TAG_HI + + /* Enable Caches before Clearing. If the caches are disabled + * then the cache operations to clear the cache will be ignored + */ + + jal enable_ID + nop + + jal size_i_cache /* v0 = i-cache size, v1 = i-cache line size */ + nop + + /* run uncached in kseg 1 */ + la k0, 1f + lui k1, 0x2000 + or k0, k1, k0 + jr k0 + nop +1: + + /* + * set K0 cache mode + */ + + mfc0 t0, CP0_CONFIG + and t0, t0, ~CP0_CONFIG_K0_MASK + or t0, t0, 3 /* Write Back mode */ + mtc0 t0, CP0_CONFIG + + /* + * Initialize instruction cache. + */ + + li a0, KSEG0 + cacheop(a0, v0, v1, Index_Store_Tag_I) + + /* + * Now we can run from I-$, kseg 0 + */ + la k0, 1f + lui k1, 0x2000 + or k0, k1, k0 + xor k0, k1, k0 + jr k0 + nop +1: + /* + * Initialize data cache. + */ + + jal size_d_cache /* v0 = d-cache size, v1 = d-cache line size */ + nop + + + li a0, KSEG0 + cacheop(a0, v0, v1, Index_Store_Tag_D) + + jr t8 + nop + + .end l1_init + .set reorder + + +/* + * Function: set_other_config + * Arguments: none + * Returns: None + * Description: initialize other remainder configuration to defaults. + * Trashes: t0, t1 + * + * pseudo code: + * + */ +LEAF(set_other_config) + .set noreorder + + /* enable Bus error for I-fetch */ + mfc0 t0, CP0_CACHEERR, 0 + li t1, 0x4 + or t0, t1 + mtc0 t0, CP0_CACHEERR, 0 + + /* enable Bus error for Load */ + mfc0 t0, CP0_CACHEERR, 1 + li t1, 0x4 + or t0, t1 + mtc0 t0, CP0_CACHEERR, 1 + + /* enable Bus Error for Store */ + mfc0 t0, CP0_CACHEERR, 2 + li t1, 0x4 + or t0, t1 + mtc0 t0, CP0_CACHEERR, 2 + + jr ra + nop + .set reorder +END(set_other_config) + +/* + * Function: set_branch_pred + * Arguments: none + * Returns: None + * Description: + * Trashes: t0, t1 + * + * pseudo code: + * + */ + +LEAF(set_branch_pred) + .set noreorder + mfc0 t0, CP0_BRCM_MODE + li t1, ~(CP0_BRCM_MODE_BrPRED_MASK | CP0_BRCM_MODE_BrHIST_MASK ) + and t0, t0, t1 + + /* enable Branch prediction */ + li t1, BRCM_BrPRED_BHT_ENABLE + sll t1, CP0_BRCM_MODE_BrPRED_SHIFT + or t0, t0, t1 + + /* set history count to 8 */ + li t1, 8 + sll t1, CP0_BRCM_MODE_BrHIST_SHIFT + or t0, t0, t1 + + mtc0 t0, CP0_BRCM_MODE + jr ra + nop + .set reorder +END(set_branch_pred) + + +/* + * Function: set_luc + * Arguments: set link uncached. + * Returns: None + * Description: + * Trashes: t0, t1 + * + */ +LEAF(set_luc) + .set noreorder + mfc0 t0, CP0_BRCM_MODE + li t1, ~(CP0_BRCM_MODE_Luc_MASK) + and t0, t0, t1 + + /* set Luc */ + ori t0, t0, CP0_BRCM_MODE_Luc_MASK + + mtc0 t0, CP0_BRCM_MODE + jr ra + nop + .set reorder +END(set_luc) + +/* + * Function: set_cwf_tse + * Arguments: set CWF and TSE bits + * Returns: None + * Description: + * Trashes: t0, t1 + * + */ +LEAF(set_cwf_tse) + .set noreorder + mfc0 t0, CP0_BRCM_CONFIG0 + li t1, (CP0_BRCM_CONFIG0_CWF_MASK | CP0_BRCM_CONFIG0_TSE_MASK) + or t0, t0, t1 + + mtc0 t0, CP0_BRCM_CONFIG0 + jr ra + nop + .set reorder +END(set_cwf_tse) + +/* + * Function: set_clock_ratio + * Arguments: set clock ratio specified by a0 + * Returns: None + * Description: + * Trashes: v0, v1, a0, a1 + * + * pseudo code: + * + */ +LEAF(set_clock_ratio) + .set noreorder + + mfc0 t0, CP0_BRCM_MODE + li t1, ~(CP0_BRCM_MODE_SET_MASK | CP0_BRCM_MODE_ClkRATIO_MASK) + and t0, t0, t1 + li t1, CP0_BRCM_MODE_SET_MASK + or t0, t0, t1 + or t0, t0, a0 + mtc0 t0, CP0_BRCM_MODE + jr ra + nop + .set reorder +END(set_clock_ratio) +/* + * Function: set_zephyr + * Arguments: None + * Returns: None + * Description: Set any zephyr bits + * Trashes: t0 & t1 + * + */ +LEAF(set_zephyr) + .set noreorder + + /* enable read/write of CP0 #22 sel. 8 */ + li t0, 0x5a455048 + .word 0x4088b00f /* mtc0 t0, $22, 15 */ + + .word 0x4008b008 /* mfc0 t0, $22, 8 */ + li t1, 0x09008000 /* turn off pref, jtb */ + or t0, t0, t1 + .word 0x4088b008 /* mtc0 t0, $22, 8 */ + sync + + /* disable read/write of CP0 #22 sel 8 */ + li t0, 0x0 + .word 0x4088b00f /* mtc0 t0, $22, 15 */ + + + jr ra + nop + .set reorder + +END(set_zephyr) + + +/* + * Function: set_llmb + * Arguments: a0=0 disable llmb, a0=1 enables llmb + * Returns: None + * Description: + * Trashes: t0, t1, t2 + * + * pseudo code: + * + */ +LEAF(set_llmb) + .set noreorder + + li t2, 0x90000000 | BRCM_ZSC_ALL_REGS_SELECT | BRCM_ZSC_CONFIG_REG + sync + cache 0x7, 0x0(t2) + sync + mfc0 t0, CP0_D_SEC_CACHE_DATA_LO + li t1, ~(BRCM_ZSC_CONFIG_LMB1En | BRCM_ZSC_CONFIG_LMB0En) + and t0, t0, t1 + + beqz a0, svlmb + nop + +enable_lmb: + li t1, (BRCM_ZSC_CONFIG_LMB1En | BRCM_ZSC_CONFIG_LMB0En) + or t0, t0, t1 + +svlmb: + mtc0 t0, CP0_D_SEC_CACHE_DATA_LO + sync + cache 0xb, 0x0(t2) + sync + + jr ra + nop + .set reorder + +END(set_llmb) +/* + * Function: core_init + * Arguments: none + * Returns: None + * Description: initialize core related configuration + * Trashes: v0,v1,a0,a1,t8 + * + * pseudo code: + * + */ + .globl core_init + .ent core_init + .set noreorder +core_init: + move t8, ra + + /* set Zephyr bits. */ + bal set_zephyr + nop + + /* set low latency memory bus */ + li a0, 1 + bal set_llmb + nop + + /* set branch prediction (TP0 only) */ + bal set_branch_pred + nop + + /* set link uncached */ + bal set_luc + nop + + /* set CWF and TSE */ + bal set_cwf_tse + nop + + /* + *set clock ratio by setting 1 to 'set' + * and 0 to ClkRatio, (TP0 only) + */ + li a0, 0 + bal set_clock_ratio + nop + + /* set other configuration to defaults */ + bal set_other_config + nop + + move ra, t8 + jr ra + nop + + .set reorder + .end core_init + +/* + * Function: clear_jump_target_buffer + * Arguments: None + * Returns: None + * Description: + * Trashes: t0, t1, t2 + * + */ +#define RESET_CALL_RETURN_STACK_THIS_THREAD (0x06<<16) +#define RESET_JUMP_TARGET_BUFFER_THIS_THREAD (0x04<<16) +#define JTB_CS_CNTL_MASK (0xFF<<16) + + .globl clear_jump_target_buffer + .ent clear_jump_target_buffer + .set noreorder +clear_jump_target_buffer: + + mfc0 t0, $22, 2 + nop + nop + + li t1, ~JTB_CS_CNTL_MASK + and t0, t0, t1 + li t2, RESET_CALL_RETURN_STACK_THIS_THREAD + or t0, t0, t2 + mtc0 t0, $22, 2 + nop + nop + + and t0, t0, t1 + li t2, RESET_JUMP_TARGET_BUFFER_THIS_THREAD + or t0, t0, t2 + mtc0 t0, $22, 2 + nop + nop + jr ra + nop + + .end clear_jump_target_buffer + .set reorder +/* + * Function: bmips_cache_init + * Arguments: None + * Returns: None + * Description: Enable I and D caches, and initialize I and D-caches + * Trashes: v0, v1, t0, t1, t2, t5, t7, t8 + * + */ + .globl bmips_5xxx_init + .ent bmips_5xxx_init + .set noreorder +bmips_5xxx_init: + + /* save return address and A0 */ + move t7, ra + move t5, a0 + + jal l1_init + nop + + jal core_init + nop + + jal clear_jump_target_buffer + nop + + mtc0 zero, CP0_CAUSE + + move a0, t5 + jr t7 + nop + + .end bmips_5xxx_init + .set reorder + + +#endif diff --git a/arch/mips/kernel/bmips_vec.S b/arch/mips/kernel/bmips_vec.S index 64c4fd62cf08..921a5fa55da6 100644 --- a/arch/mips/kernel/bmips_vec.S +++ b/arch/mips/kernel/bmips_vec.S @@ -8,11 +8,11 @@ * Reset/NMI/re-entry vectors for BMIPS processors */ -#include <linux/init.h> #include <asm/asm.h> #include <asm/asmmacro.h> #include <asm/cacheops.h> +#include <asm/cpu.h> #include <asm/regdef.h> #include <asm/mipsregs.h> #include <asm/stackframe.h> @@ -28,8 +28,6 @@ .set mips0 .endm - __CPUINIT - /*********************************************************************** * Alternate CPU1 startup vector for BMIPS4350 * @@ -56,7 +54,11 @@ LEAF(bmips_smp_movevec) /* set up CPU1 CBR; move BASE to 0xa000_0000 */ li k0, 0xff400000 mtc0 k0, $22, 6 - li k1, CKSEG1 | BMIPS_RELO_VECTOR_CONTROL_1 + /* set up relocation vector address based on thread ID */ + mfc0 k1, $22, 3 + srl k1, 16 + andi k1, 0x8000 + or k1, CKSEG1 | BMIPS_RELO_VECTOR_CONTROL_0 or k0, k1 li k1, 0xa0080000 sw k1, 0(k0) @@ -86,15 +88,22 @@ NESTED(bmips_reset_nmi_vec, PT_SIZE, sp) li k1, (1 << 19) mfc0 k0, CP0_STATUS and k0, k1 - beqz k0, bmips_smp_entry + beqz k0, soft_reset #if defined(CONFIG_CPU_BMIPS5000) + mfc0 k0, CP0_PRID + li k1, PRID_IMP_BMIPS5000 + /* mask with PRID_IMP_BMIPS5000 to cover both variants */ + andi k0, PRID_IMP_BMIPS5000 + bne k0, k1, 1f + /* if we're not on core 0, this must be the SMP boot signal */ li k1, (3 << 25) mfc0 k0, $22 and k0, k1 bnez k0, bmips_smp_entry -#endif +1: +#endif /* CONFIG_CPU_BMIPS5000 */ #endif /* CONFIG_SMP */ /* nope, it's just a regular NMI */ @@ -114,16 +123,51 @@ NESTED(bmips_reset_nmi_vec, PT_SIZE, sp) jr k0 RESTORE_ALL - .set mips3 + .set arch=r4000 eret +#ifdef CONFIG_SMP +soft_reset: + +#if defined(CONFIG_CPU_BMIPS5000) + mfc0 k0, CP0_PRID + andi k0, 0xff00 + li k1, PRID_IMP_BMIPS5200 + bne k0, k1, bmips_smp_entry + + /* if running on TP 1, jump to bmips_smp_entry */ + mfc0 k0, $22 + li k1, (1 << 24) + and k1, k0 + bnez k1, bmips_smp_entry + nop + + /* + * running on TP0, can not be core 0 (the boot core). + * Check for soft reset. Indicates a warm boot + */ + mfc0 k0, $12 + li k1, (1 << 20) + and k0, k1 + beqz k0, bmips_smp_entry + + /* + * Warm boot. + * Cache init is only done on TP0 + */ + la k0, bmips_5xxx_init + jalr k0 + nop + + b bmips_smp_entry + nop +#endif + /*********************************************************************** * CPU1 reset vector (used for the initial boot only) * This is still part of bmips_reset_nmi_vec(). ***********************************************************************/ -#ifdef CONFIG_SMP - bmips_smp_entry: /* set up CP0 STATUS; enable FPU */ @@ -137,7 +181,12 @@ bmips_smp_entry: xori k0, 0x04 mtc0 k0, CP0_CONFIG + mfc0 k0, CP0_PRID + andi k0, 0xff00 #if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380) + li k1, PRID_IMP_BMIPS43XX + bne k0, k1, 2f + /* initialize CPU1's local I-cache */ li k0, 0x80000000 li k1, 0x80010000 @@ -148,14 +197,23 @@ bmips_smp_entry: 1: cache Index_Store_Tag_I, 0(k0) addiu k0, 16 bne k0, k1, 1b -#elif defined(CONFIG_CPU_BMIPS5000) + + b 3f +2: +#endif /* CONFIG_CPU_BMIPS4350 || CONFIG_CPU_BMIPS4380 */ +#if defined(CONFIG_CPU_BMIPS5000) + /* mask with PRID_IMP_BMIPS5000 to cover both variants */ + li k1, PRID_IMP_BMIPS5000 + andi k0, PRID_IMP_BMIPS5000 + bne k0, k1, 3f + /* set exception vector base */ la k0, ebase lw k0, 0(k0) mtc0 k0, $15, 1 BARRIER -#endif - +#endif /* CONFIG_CPU_BMIPS5000 */ +3: /* jump back to kseg0 in case we need to remap the kseg1 area */ la k0, 1f jr k0 @@ -188,7 +246,6 @@ bmips_reset_nmi_vec_end: END(bmips_reset_nmi_vec) .set pop - .previous /*********************************************************************** * CPU1 warm restart vector (used for second and subsequent boots). @@ -216,13 +273,21 @@ END(bmips_smp_int_vec) * Certain CPUs support extending kseg0 to 1024MB. ***********************************************************************/ - __CPUINIT - LEAF(bmips_enable_xks01) #if defined(CONFIG_XKS01) - + mfc0 t0, CP0_PRID + andi t2, t0, 0xff00 #if defined(CONFIG_CPU_BMIPS4380) + li t1, PRID_IMP_BMIPS43XX + bne t2, t1, 1f + + andi t0, 0xff + addiu t1, t0, -PRID_REV_BMIPS4380_HI + bgtz t1, 2f + addiu t0, -PRID_REV_BMIPS4380_LO + bltz t0, 2f + mfc0 t0, $22, 3 li t1, 0x1ff0 li t2, (1 << 12) | (1 << 9) @@ -231,7 +296,15 @@ LEAF(bmips_enable_xks01) or t0, t2 mtc0 t0, $22, 3 BARRIER -#elif defined(CONFIG_CPU_BMIPS5000) + b 2f +1: +#endif /* CONFIG_CPU_BMIPS4380 */ +#if defined(CONFIG_CPU_BMIPS5000) + li t1, PRID_IMP_BMIPS5000 + /* mask with PRID_IMP_BMIPS5000 to cover both variants */ + andi t2, PRID_IMP_BMIPS5000 + bne t2, t1, 2f + mfc0 t0, $22, 5 li t1, 0x01ff li t2, (1 << 8) | (1 << 5) @@ -240,16 +313,10 @@ LEAF(bmips_enable_xks01) or t0, t2 mtc0 t0, $22, 5 BARRIER -#else - -#error Missing XKS01 setup - -#endif - +#endif /* CONFIG_CPU_BMIPS5000 */ +2: #endif /* defined(CONFIG_XKS01) */ jr ra END(bmips_enable_xks01) - - .previous diff --git a/arch/mips/kernel/branch.c b/arch/mips/kernel/branch.c index 4d78bf445a9c..0216ff24c392 100644 --- a/arch/mips/kernel/branch.c +++ b/arch/mips/kernel/branch.c @@ -7,17 +7,20 @@ * Copyright (C) 2001 MIPS Technologies, Inc. */ #include <linux/kernel.h> -#include <linux/sched.h> +#include <linux/sched/signal.h> #include <linux/signal.h> -#include <linux/module.h> +#include <linux/export.h> #include <asm/branch.h> #include <asm/cpu.h> #include <asm/cpu-features.h> #include <asm/fpu.h> #include <asm/fpu_emulator.h> #include <asm/inst.h> +#include <asm/mips-r2-to-r6-emul.h> #include <asm/ptrace.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> + +#include "probes-common.h" /* * Calculate and return exception PC in case of branch delay slot @@ -31,12 +34,14 @@ int __isa_exception_epc(struct pt_regs *regs) /* Calculate exception PC in branch delay slot. */ if (__get_user(inst, (u16 __user *) msk_isa16_mode(epc))) { /* This should never happen because delay slot was checked. */ - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); return epc; } if (cpu_has_mips16) { - if (((union mips16e_instruction)inst).ri.opcode - == MIPS16e_jal_op) + union mips16e_instruction inst_mips16e; + + inst_mips16e.full = inst; + if (inst_mips16e.ri.opcode == MIPS16e_jal_op) epc += 4; else epc += 2; @@ -48,6 +53,206 @@ int __isa_exception_epc(struct pt_regs *regs) return epc; } +/* (microMIPS) Convert 16-bit register encoding to 32-bit register encoding. */ +static const unsigned int reg16to32map[8] = {16, 17, 2, 3, 4, 5, 6, 7}; + +int __mm_isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, + unsigned long *contpc) +{ + union mips_instruction insn = (union mips_instruction)dec_insn.insn; + int __maybe_unused bc_false = 0; + + if (!cpu_has_mmips) + return 0; + + switch (insn.mm_i_format.opcode) { + case mm_pool32a_op: + if ((insn.mm_i_format.simmediate & MM_POOL32A_MINOR_MASK) == + mm_pool32axf_op) { + switch (insn.mm_i_format.simmediate >> + MM_POOL32A_MINOR_SHIFT) { + case mm_jalr_op: + case mm_jalrhb_op: + case mm_jalrs_op: + case mm_jalrshb_op: + if (insn.mm_i_format.rt != 0) /* Not mm_jr */ + regs->regs[insn.mm_i_format.rt] = + regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + *contpc = regs->regs[insn.mm_i_format.rs]; + return 1; + } + } + break; + case mm_pool32i_op: + switch (insn.mm_i_format.rt) { + case mm_bltzals_op: + case mm_bltzal_op: + regs->regs[31] = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + fallthrough; + case mm_bltz_op: + if ((long)regs->regs[insn.mm_i_format.rs] < 0) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + return 1; + case mm_bgezals_op: + case mm_bgezal_op: + regs->regs[31] = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + fallthrough; + case mm_bgez_op: + if ((long)regs->regs[insn.mm_i_format.rs] >= 0) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + return 1; + case mm_blez_op: + if ((long)regs->regs[insn.mm_i_format.rs] <= 0) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + return 1; + case mm_bgtz_op: + if ((long)regs->regs[insn.mm_i_format.rs] <= 0) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + return 1; +#ifdef CONFIG_MIPS_FP_SUPPORT + case mm_bc2f_op: + case mm_bc1f_op: { + unsigned int fcr31; + unsigned int bit; + + bc_false = 1; + fallthrough; + case mm_bc2t_op: + case mm_bc1t_op: + preempt_disable(); + if (is_fpu_owner()) + fcr31 = read_32bit_cp1_register(CP1_STATUS); + else + fcr31 = current->thread.fpu.fcr31; + preempt_enable(); + + if (bc_false) + fcr31 = ~fcr31; + + bit = (insn.mm_i_format.rs >> 2); + bit += (bit != 0); + bit += 23; + if (fcr31 & (1 << bit)) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + return 1; + } +#endif /* CONFIG_MIPS_FP_SUPPORT */ + } + break; + case mm_pool16c_op: + switch (insn.mm_i_format.rt) { + case mm_jalr16_op: + case mm_jalrs16_op: + regs->regs[31] = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + fallthrough; + case mm_jr16_op: + *contpc = regs->regs[insn.mm_i_format.rs]; + return 1; + } + break; + case mm_beqz16_op: + if ((long)regs->regs[reg16to32map[insn.mm_b1_format.rs]] == 0) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_b1_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + return 1; + case mm_bnez16_op: + if ((long)regs->regs[reg16to32map[insn.mm_b1_format.rs]] != 0) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_b1_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + return 1; + case mm_b16_op: + *contpc = regs->cp0_epc + dec_insn.pc_inc + + (insn.mm_b0_format.simmediate << 1); + return 1; + case mm_beq32_op: + if (regs->regs[insn.mm_i_format.rs] == + regs->regs[insn.mm_i_format.rt]) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + return 1; + case mm_bne32_op: + if (regs->regs[insn.mm_i_format.rs] != + regs->regs[insn.mm_i_format.rt]) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + return 1; + case mm_jalx32_op: + regs->regs[31] = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + *contpc = regs->cp0_epc + dec_insn.pc_inc; + *contpc >>= 28; + *contpc <<= 28; + *contpc |= (insn.j_format.target << 2); + return 1; + case mm_jals32_op: + case mm_jal32_op: + regs->regs[31] = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + fallthrough; + case mm_j32_op: + *contpc = regs->cp0_epc + dec_insn.pc_inc; + *contpc >>= 27; + *contpc <<= 27; + *contpc |= (insn.j_format.target << 1); + set_isa16_mode(*contpc); + return 1; + } + return 0; +} + /* * Compute return address and emulate branch in microMIPS mode after an * exception only. It does not handle compact branches/jumps and cannot @@ -102,7 +307,7 @@ int __microMIPS_compute_return_epc(struct pt_regs *regs) return 0; sigsegv: - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); return -EFAULT; } @@ -125,7 +330,7 @@ int __MIPS16e_compute_return_epc(struct pt_regs *regs) /* Read the instruction. */ addr = (u16 __user *)msk_isa16_mode(epc); if (__get_user(inst.full, addr)) { - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); return -EFAULT; } @@ -140,7 +345,7 @@ int __MIPS16e_compute_return_epc(struct pt_regs *regs) case MIPS16e_jal_op: addr += 1; if (__get_user(inst2, addr)) { - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); return -EFAULT; } fullinst = ((unsigned)inst.full << 16) | inst2; @@ -200,15 +405,25 @@ int __MIPS16e_compute_return_epc(struct pt_regs *regs) * * @regs: Pointer to pt_regs * @insn: branch instruction to decode - * @returns: -EFAULT on error and forces SIGBUS, and on success + * Return: -EFAULT on error and forces SIGILL, and on success * returns 0 or BRANCH_LIKELY_TAKEN as appropriate after * evaluating the branch. + * + * MIPS R6 Compact branches and forbidden slots: + * Compact branches do not throw exceptions because they do + * not have delay slots. The forbidden slot instruction ($PC+4) + * is only executed if the branch was not taken. Otherwise the + * forbidden slot is skipped entirely. This means that the + * only possible reason to be here because of a MIPS R6 compact + * branch instruction is that the forbidden slot has thrown one. + * In that case the branch was not taken, so the EPC can be safely + * set to EPC + 8. */ int __compute_return_epc_for_insn(struct pt_regs *regs, union mips_instruction insn) { - unsigned int bit, fcr31, dspcontrol; long epc = regs->cp0_epc; + unsigned int dspcontrol; int ret = 0; switch (insn.i_format.opcode) { @@ -219,8 +434,10 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, switch (insn.r_format.func) { case jalr_op: regs->regs[insn.r_format.rd] = epc + 8; - /* Fall through */ + fallthrough; case jr_op: + if (NO_R6EMU && insn.r_format.func == jr_op) + goto sigill_r2r6; regs->cp0_epc = regs->regs[insn.r_format.rs]; break; } @@ -233,8 +450,11 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, */ case bcond_op: switch (insn.i_format.rt) { - case bltz_op: case bltzl_op: + if (NO_R6EMU) + goto sigill_r2r6; + fallthrough; + case bltz_op: if ((long)regs->regs[insn.i_format.rs] < 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); if (insn.i_format.rt == bltzl_op) @@ -244,8 +464,11 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, regs->cp0_epc = epc; break; - case bgez_op: case bgezl_op: + if (NO_R6EMU) + goto sigill_r2r6; + fallthrough; + case bgez_op: if ((long)regs->regs[insn.i_format.rs] >= 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); if (insn.i_format.rt == bgezl_op) @@ -257,7 +480,27 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, case bltzal_op: case bltzall_op: + if (NO_R6EMU && (insn.i_format.rs || + insn.i_format.rt == bltzall_op)) + goto sigill_r2r6; regs->regs[31] = epc + 8; + /* + * OK we are here either because we hit a NAL + * instruction or because we are emulating an + * old bltzal{,l} one. Let's figure out what the + * case really is. + */ + if (!insn.i_format.rs) { + /* + * NAL or BLTZAL with rs == 0 + * Doesn't matter if we are R6 or not. The + * result is the same + */ + regs->cp0_epc += 4 + + (insn.i_format.simmediate << 2); + break; + } + /* Now do the real thing for non-R6 BLTZAL{,L} */ if ((long)regs->regs[insn.i_format.rs] < 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); if (insn.i_format.rt == bltzall_op) @@ -269,7 +512,27 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, case bgezal_op: case bgezall_op: + if (NO_R6EMU && (insn.i_format.rs || + insn.i_format.rt == bgezall_op)) + goto sigill_r2r6; regs->regs[31] = epc + 8; + /* + * OK we are here either because we hit a BAL + * instruction or because we are emulating an + * old bgezal{,l} one. Let's figure out what the + * case really is. + */ + if (!insn.i_format.rs) { + /* + * BAL or BGEZAL with rs == 0 + * Doesn't matter if we are R6 or not. The + * result is the same + */ + regs->cp0_epc += 4 + + (insn.i_format.simmediate << 2); + break; + } + /* Now do the real thing for non-R6 BGEZAL{,L} */ if ((long)regs->regs[insn.i_format.rs] >= 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); if (insn.i_format.rt == bgezall_op) @@ -281,7 +544,7 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, case bposge32_op: if (!cpu_has_dsp) - goto sigill; + goto sigill_dsp; dspcontrol = rddsp(0x01); @@ -297,8 +560,10 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, /* * These are unconditional and in j_format. */ + case jalx_op: case jal_op: regs->regs[31] = regs->cp0_epc + 8; + fallthrough; case j_op: epc += 4; epc >>= 28; @@ -312,92 +577,176 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, /* * These are conditional and in i_format. */ - case beq_op: case beql_op: + if (NO_R6EMU) + goto sigill_r2r6; + fallthrough; + case beq_op: if (regs->regs[insn.i_format.rs] == regs->regs[insn.i_format.rt]) { epc = epc + 4 + (insn.i_format.simmediate << 2); - if (insn.i_format.rt == beql_op) + if (insn.i_format.opcode == beql_op) ret = BRANCH_LIKELY_TAKEN; } else epc += 8; regs->cp0_epc = epc; break; - case bne_op: case bnel_op: + if (NO_R6EMU) + goto sigill_r2r6; + fallthrough; + case bne_op: if (regs->regs[insn.i_format.rs] != regs->regs[insn.i_format.rt]) { epc = epc + 4 + (insn.i_format.simmediate << 2); - if (insn.i_format.rt == bnel_op) + if (insn.i_format.opcode == bnel_op) ret = BRANCH_LIKELY_TAKEN; } else epc += 8; regs->cp0_epc = epc; break; - case blez_op: /* not really i_format */ - case blezl_op: + case blezl_op: /* not really i_format */ + if (!insn.i_format.rt && NO_R6EMU) + goto sigill_r2r6; + fallthrough; + case blez_op: + /* + * Compact branches for R6 for the + * blez and blezl opcodes. + * BLEZ | rs = 0 | rt != 0 == BLEZALC + * BLEZ | rs = rt != 0 == BGEZALC + * BLEZ | rs != 0 | rt != 0 == BGEUC + * BLEZL | rs = 0 | rt != 0 == BLEZC + * BLEZL | rs = rt != 0 == BGEZC + * BLEZL | rs != 0 | rt != 0 == BGEC + * + * For real BLEZ{,L}, rt is always 0. + */ + + if (cpu_has_mips_r6 && insn.i_format.rt) { + if ((insn.i_format.opcode == blez_op) && + ((!insn.i_format.rs && insn.i_format.rt) || + (insn.i_format.rs == insn.i_format.rt))) + regs->regs[31] = epc + 4; + regs->cp0_epc += 8; + break; + } /* rt field assumed to be zero */ if ((long)regs->regs[insn.i_format.rs] <= 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); - if (insn.i_format.rt == bnel_op) + if (insn.i_format.opcode == blezl_op) ret = BRANCH_LIKELY_TAKEN; } else epc += 8; regs->cp0_epc = epc; break; - case bgtz_op: case bgtzl_op: + if (!insn.i_format.rt && NO_R6EMU) + goto sigill_r2r6; + fallthrough; + case bgtz_op: + /* + * Compact branches for R6 for the + * bgtz and bgtzl opcodes. + * BGTZ | rs = 0 | rt != 0 == BGTZALC + * BGTZ | rs = rt != 0 == BLTZALC + * BGTZ | rs != 0 | rt != 0 == BLTUC + * BGTZL | rs = 0 | rt != 0 == BGTZC + * BGTZL | rs = rt != 0 == BLTZC + * BGTZL | rs != 0 | rt != 0 == BLTC + * + * *ZALC varint for BGTZ &&& rt != 0 + * For real GTZ{,L}, rt is always 0. + */ + if (cpu_has_mips_r6 && insn.i_format.rt) { + if ((insn.i_format.opcode == blez_op) && + ((!insn.i_format.rs && insn.i_format.rt) || + (insn.i_format.rs == insn.i_format.rt))) + regs->regs[31] = epc + 4; + regs->cp0_epc += 8; + break; + } + /* rt field assumed to be zero */ if ((long)regs->regs[insn.i_format.rs] > 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); - if (insn.i_format.rt == bnel_op) + if (insn.i_format.opcode == bgtzl_op) ret = BRANCH_LIKELY_TAKEN; } else epc += 8; regs->cp0_epc = epc; break; +#ifdef CONFIG_MIPS_FP_SUPPORT /* * And now the FPA/cp1 branch instructions. */ - case cop1_op: - preempt_disable(); - if (is_fpu_owner()) - asm volatile("cfc1\t%0,$31" : "=r" (fcr31)); - else - fcr31 = current->thread.fpu.fcr31; - preempt_enable(); - - bit = (insn.i_format.rt >> 2); - bit += (bit != 0); - bit += 23; - switch (insn.i_format.rt & 3) { - case 0: /* bc1f */ - case 2: /* bc1fl */ - if (~fcr31 & (1 << bit)) { - epc = epc + 4 + (insn.i_format.simmediate << 2); - if (insn.i_format.rt == 2) - ret = BRANCH_LIKELY_TAKEN; - } else + case cop1_op: { + unsigned int bit, fcr31, reg; + + if (cpu_has_mips_r6 && + ((insn.i_format.rs == bc1eqz_op) || + (insn.i_format.rs == bc1nez_op))) { + if (!init_fp_ctx(current)) + lose_fpu(1); + reg = insn.i_format.rt; + bit = get_fpr32(¤t->thread.fpu.fpr[reg], 0) & 0x1; + if (insn.i_format.rs == bc1eqz_op) + bit = !bit; + own_fpu(1); + if (bit) + epc = epc + 4 + + (insn.i_format.simmediate << 2); + else epc += 8; regs->cp0_epc = epc; + break; + } else { - case 1: /* bc1t */ - case 3: /* bc1tl */ - if (fcr31 & (1 << bit)) { - epc = epc + 4 + (insn.i_format.simmediate << 2); - if (insn.i_format.rt == 3) - ret = BRANCH_LIKELY_TAKEN; - } else - epc += 8; - regs->cp0_epc = epc; + preempt_disable(); + if (is_fpu_owner()) + fcr31 = read_32bit_cp1_register(CP1_STATUS); + else + fcr31 = current->thread.fpu.fcr31; + preempt_enable(); + + bit = (insn.i_format.rt >> 2); + bit += (bit != 0); + bit += 23; + switch (insn.i_format.rt & 3) { + case 0: /* bc1f */ + case 2: /* bc1fl */ + if (~fcr31 & (1 << bit)) { + epc = epc + 4 + + (insn.i_format.simmediate << 2); + if (insn.i_format.rt == 2) + ret = BRANCH_LIKELY_TAKEN; + } else + epc += 8; + regs->cp0_epc = epc; + break; + + case 1: /* bc1t */ + case 3: /* bc1tl */ + if (fcr31 & (1 << bit)) { + epc = epc + 4 + + (insn.i_format.simmediate << 2); + if (insn.i_format.rt == 3) + ret = BRANCH_LIKELY_TAKEN; + } else + epc += 8; + regs->cp0_epc = epc; + break; + } break; } - break; + } +#endif /* CONFIG_MIPS_FP_SUPPORT */ + #ifdef CONFIG_CPU_CAVIUM_OCTEON case lwc2_op: /* This is bbit0 on Octeon */ if ((regs->regs[insn.i_format.rs] & (1ull<<insn.i_format.rt)) @@ -430,14 +779,69 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, epc += 8; regs->cp0_epc = epc; break; +#else + case bc6_op: + /* Only valid for MIPS R6 */ + if (!cpu_has_mips_r6) + goto sigill_r6; + regs->cp0_epc += 8; + break; + case balc6_op: + if (!cpu_has_mips_r6) + goto sigill_r6; + /* Compact branch: BALC */ + regs->regs[31] = epc + 4; + epc += 4 + (insn.i_format.simmediate << 2); + regs->cp0_epc = epc; + break; + case pop66_op: + if (!cpu_has_mips_r6) + goto sigill_r6; + /* Compact branch: BEQZC || JIC */ + regs->cp0_epc += 8; + break; + case pop76_op: + if (!cpu_has_mips_r6) + goto sigill_r6; + /* Compact branch: BNEZC || JIALC */ + if (!insn.i_format.rs) { + /* JIALC: set $31/ra */ + regs->regs[31] = epc + 4; + } + regs->cp0_epc += 8; + break; #endif + case pop10_op: + case pop30_op: + /* Only valid for MIPS R6 */ + if (!cpu_has_mips_r6) + goto sigill_r6; + /* + * Compact branches: + * bovc, beqc, beqzalc, bnvc, bnec, bnezlac + */ + if (insn.i_format.rt && !insn.i_format.rs) + regs->regs[31] = epc + 4; + regs->cp0_epc += 8; + break; } return ret; -sigill: - printk("%s: DSP branch but not DSP ASE - sending SIGBUS.\n", current->comm); - force_sig(SIGBUS, current); +sigill_dsp: + pr_debug("%s: DSP branch but not DSP ASE - sending SIGILL.\n", + current->comm); + force_sig(SIGILL); + return -EFAULT; +sigill_r2r6: + pr_debug("%s: R2 branch but r2-to-r6 emulator is not present - sending SIGILL.\n", + current->comm); + force_sig(SIGILL); + return -EFAULT; +sigill_r6: + pr_debug("%s: R6 branch but no MIPSr6 ISA support - sending SIGILL.\n", + current->comm); + force_sig(SIGILL); return -EFAULT; } EXPORT_SYMBOL_GPL(__compute_return_epc_for_insn); @@ -457,7 +861,7 @@ int __compute_return_epc(struct pt_regs *regs) */ addr = (unsigned int __user *) epc; if (__get_user(insn.word, addr)) { - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); return -EFAULT; } @@ -465,6 +869,40 @@ int __compute_return_epc(struct pt_regs *regs) unaligned: printk("%s: unaligned epc - sending SIGBUS.\n", current->comm); - force_sig(SIGBUS, current); + force_sig(SIGBUS); return -EFAULT; } + +#if (defined CONFIG_KPROBES) || (defined CONFIG_UPROBES) + +int __insn_is_compact_branch(union mips_instruction insn) +{ + if (!cpu_has_mips_r6) + return 0; + + switch (insn.i_format.opcode) { + case blezl_op: + case bgtzl_op: + case blez_op: + case bgtz_op: + /* + * blez[l] and bgtz[l] opcodes with non-zero rt + * are MIPS R6 compact branches + */ + if (insn.i_format.rt) + return 1; + break; + case bc6_op: + case balc6_op: + case pop10_op: + case pop30_op: + case pop66_op: + case pop76_op: + return 1; + } + + return 0; +} +EXPORT_SYMBOL_GPL(__insn_is_compact_branch); + +#endif /* CONFIG_KPROBES || CONFIG_UPROBES */ diff --git a/arch/mips/kernel/cacheinfo.c b/arch/mips/kernel/cacheinfo.c new file mode 100644 index 000000000000..495dd058231d --- /dev/null +++ b/arch/mips/kernel/cacheinfo.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * MIPS cacheinfo support + */ +#include <linux/cacheinfo.h> + +/* Populates leaf and increments to next leaf */ +#define populate_cache(cache, leaf, c_level, c_type) \ +do { \ + leaf->type = c_type; \ + leaf->level = c_level; \ + leaf->coherency_line_size = c->cache.linesz; \ + leaf->number_of_sets = c->cache.sets; \ + leaf->ways_of_associativity = c->cache.ways; \ + leaf->size = c->cache.linesz * c->cache.sets * \ + c->cache.ways; \ + leaf++; \ +} while (0) + +int init_cache_level(unsigned int cpu) +{ + struct cpuinfo_mips *c = ¤t_cpu_data; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + int levels = 0, leaves = 0; + + /* + * If Dcache is not set, we assume the cache structures + * are not properly initialized. + */ + if (c->dcache.waysize) + levels += 1; + else + return -ENOENT; + + + leaves += (c->icache.waysize) ? 2 : 1; + + if (c->vcache.waysize) { + levels++; + leaves++; + } + + if (c->scache.waysize) { + levels++; + leaves++; + } + + if (c->tcache.waysize) { + levels++; + leaves++; + } + + this_cpu_ci->num_levels = levels; + this_cpu_ci->num_leaves = leaves; + return 0; +} + +static void fill_cpumask_siblings(int cpu, cpumask_t *cpu_map) +{ + int cpu1; + + for_each_possible_cpu(cpu1) + if (cpus_are_siblings(cpu, cpu1)) + cpumask_set_cpu(cpu1, cpu_map); +} + +static void fill_cpumask_cluster(int cpu, cpumask_t *cpu_map) +{ + int cpu1; + int cluster = cpu_cluster(&cpu_data[cpu]); + + for_each_possible_cpu(cpu1) + if (cpu_cluster(&cpu_data[cpu1]) == cluster) + cpumask_set_cpu(cpu1, cpu_map); +} + +int populate_cache_leaves(unsigned int cpu) +{ + struct cpuinfo_mips *c = ¤t_cpu_data; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct cacheinfo *this_leaf = this_cpu_ci->info_list; + int level = 1; + + if (c->icache.waysize) { + /* I/D caches are per core */ + fill_cpumask_siblings(cpu, &this_leaf->shared_cpu_map); + populate_cache(dcache, this_leaf, level, CACHE_TYPE_DATA); + fill_cpumask_siblings(cpu, &this_leaf->shared_cpu_map); + populate_cache(icache, this_leaf, level, CACHE_TYPE_INST); + level++; + } else { + populate_cache(dcache, this_leaf, level, CACHE_TYPE_UNIFIED); + level++; + } + + if (c->vcache.waysize) { + /* Vcache is per core as well */ + fill_cpumask_siblings(cpu, &this_leaf->shared_cpu_map); + populate_cache(vcache, this_leaf, level, CACHE_TYPE_UNIFIED); + level++; + } + + if (c->scache.waysize) { + /* Scache is per cluster */ + fill_cpumask_cluster(cpu, &this_leaf->shared_cpu_map); + populate_cache(scache, this_leaf, level, CACHE_TYPE_UNIFIED); + level++; + } + + if (c->tcache.waysize) + populate_cache(tcache, this_leaf, level, CACHE_TYPE_UNIFIED); + + this_cpu_ci->cpu_map_populated = true; + + return 0; +} diff --git a/arch/mips/kernel/cevt-bcm1480.c b/arch/mips/kernel/cevt-bcm1480.c index 15f618b40cf6..2a14dc4ee57e 100644 --- a/arch/mips/kernel/cevt-bcm1480.c +++ b/arch/mips/kernel/cevt-bcm1480.c @@ -1,19 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2000,2001,2004 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ #include <linux/clockchips.h> #include <linux/interrupt.h> @@ -40,8 +27,8 @@ * The general purpose timer ticks at 1MHz independent if * the rest of the system */ -static void sibyte_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) + +static int sibyte_set_periodic(struct clock_event_device *evt) { unsigned int cpu = smp_processor_id(); void __iomem *cfg, *init; @@ -49,24 +36,22 @@ static void sibyte_set_mode(enum clock_event_mode mode, cfg = IOADDR(A_SCD_TIMER_REGISTER(cpu, R_SCD_TIMER_CFG)); init = IOADDR(A_SCD_TIMER_REGISTER(cpu, R_SCD_TIMER_INIT)); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - __raw_writeq(0, cfg); - __raw_writeq((V_SCD_TIMER_FREQ / HZ) - 1, init); - __raw_writeq(M_SCD_TIMER_ENABLE | M_SCD_TIMER_MODE_CONTINUOUS, - cfg); - break; - - case CLOCK_EVT_MODE_ONESHOT: - /* Stop the timer until we actually program a shot */ - case CLOCK_EVT_MODE_SHUTDOWN: - __raw_writeq(0, cfg); - break; - - case CLOCK_EVT_MODE_UNUSED: /* shuddup gcc */ - case CLOCK_EVT_MODE_RESUME: - ; - } + __raw_writeq(0, cfg); + __raw_writeq((V_SCD_TIMER_FREQ / HZ) - 1, init); + __raw_writeq(M_SCD_TIMER_ENABLE | M_SCD_TIMER_MODE_CONTINUOUS, cfg); + return 0; +} + +static int sibyte_shutdown(struct clock_event_device *evt) +{ + unsigned int cpu = smp_processor_id(); + void __iomem *cfg; + + cfg = IOADDR(A_SCD_TIMER_REGISTER(cpu, R_SCD_TIMER_CFG)); + + /* Stop the timer until we actually program a shot */ + __raw_writeq(0, cfg); + return 0; } static int sibyte_next_event(unsigned long delta, struct clock_event_device *cd) @@ -91,7 +76,7 @@ static irqreturn_t sibyte_counter_handler(int irq, void *dev_id) void __iomem *cfg; unsigned long tmode; - if (cd->mode == CLOCK_EVT_MODE_PERIODIC) + if (clockevent_state_periodic(cd)) tmode = M_SCD_TIMER_ENABLE | M_SCD_TIMER_MODE_CONTINUOUS; else tmode = 0; @@ -106,31 +91,34 @@ static irqreturn_t sibyte_counter_handler(int irq, void *dev_id) } static DEFINE_PER_CPU(struct clock_event_device, sibyte_hpt_clockevent); -static DEFINE_PER_CPU(struct irqaction, sibyte_hpt_irqaction); static DEFINE_PER_CPU(char [18], sibyte_hpt_name); -void __cpuinit sb1480_clockevent_init(void) +void sb1480_clockevent_init(void) { unsigned int cpu = smp_processor_id(); unsigned int irq = K_BCM1480_INT_TIMER_0 + cpu; - struct irqaction *action = &per_cpu(sibyte_hpt_irqaction, cpu); struct clock_event_device *cd = &per_cpu(sibyte_hpt_clockevent, cpu); unsigned char *name = per_cpu(sibyte_hpt_name, cpu); + unsigned long flags = IRQF_PERCPU | IRQF_TIMER; BUG_ON(cpu > 3); /* Only have 4 general purpose timers */ - sprintf(name, "bcm1480-counter-%d", cpu); + sprintf(name, "bcm1480-counter-%u", cpu); cd->name = name; cd->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; clockevent_set_clock(cd, V_SCD_TIMER_FREQ); cd->max_delta_ns = clockevent_delta2ns(0x7fffff, cd); + cd->max_delta_ticks = 0x7fffff; cd->min_delta_ns = clockevent_delta2ns(2, cd); + cd->min_delta_ticks = 2; cd->rating = 200; cd->irq = irq; cd->cpumask = cpumask_of(cpu); cd->set_next_event = sibyte_next_event; - cd->set_mode = sibyte_set_mode; + cd->set_state_shutdown = sibyte_shutdown; + cd->set_state_periodic = sibyte_set_periodic; + cd->set_state_oneshot = sibyte_shutdown; clockevents_register_device(cd); bcm1480_mask_irq(cpu, irq); @@ -144,11 +132,7 @@ void __cpuinit sb1480_clockevent_init(void) bcm1480_unmask_irq(cpu, irq); - action->handler = sibyte_counter_handler; - action->flags = IRQF_PERCPU | IRQF_TIMER; - action->name = name; - action->dev_id = cd; - irq_set_affinity(irq, cpumask_of(cpu)); - setup_irq(irq, action); + if (request_irq(irq, sibyte_counter_handler, flags, name, cd)) + pr_err("Failed to request irq %d (%s)\n", irq, name); } diff --git a/arch/mips/kernel/cevt-ds1287.c b/arch/mips/kernel/cevt-ds1287.c index ff1f01b72270..de64d6bb7ba3 100644 --- a/arch/mips/kernel/cevt-ds1287.c +++ b/arch/mips/kernel/cevt-ds1287.c @@ -1,21 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * DS1287 clockevent driver * * Copyright (C) 2008 Yoichi Yuasa <yuasa@linux-mips.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include <linux/clockchips.h> #include <linux/init.h> @@ -23,6 +10,7 @@ #include <linux/mc146818rtc.h> #include <linux/irq.h> +#include <asm/ds1287.h> #include <asm/time.h> int ds1287_timer_state(void) @@ -59,27 +47,32 @@ static int ds1287_set_next_event(unsigned long delta, return -EINVAL; } -static void ds1287_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int ds1287_shutdown(struct clock_event_device *evt) { u8 val; spin_lock(&rtc_lock); val = CMOS_READ(RTC_REG_B); + val &= ~RTC_PIE; + CMOS_WRITE(val, RTC_REG_B); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - val |= RTC_PIE; - break; - default: - val &= ~RTC_PIE; - break; - } + spin_unlock(&rtc_lock); + return 0; +} + +static int ds1287_set_periodic(struct clock_event_device *evt) +{ + u8 val; + + spin_lock(&rtc_lock); + val = CMOS_READ(RTC_REG_B); + val |= RTC_PIE; CMOS_WRITE(val, RTC_REG_B); spin_unlock(&rtc_lock); + return 0; } static void ds1287_event_handler(struct clock_event_device *dev) @@ -87,11 +80,13 @@ static void ds1287_event_handler(struct clock_event_device *dev) } static struct clock_event_device ds1287_clockevent = { - .name = "ds1287", - .features = CLOCK_EVT_FEAT_PERIODIC, - .set_next_event = ds1287_set_next_event, - .set_mode = ds1287_set_mode, - .event_handler = ds1287_event_handler, + .name = "ds1287", + .features = CLOCK_EVT_FEAT_PERIODIC, + .set_next_event = ds1287_set_next_event, + .set_state_shutdown = ds1287_shutdown, + .set_state_periodic = ds1287_set_periodic, + .tick_resume = ds1287_shutdown, + .event_handler = ds1287_event_handler, }; static irqreturn_t ds1287_interrupt(int irq, void *dev_id) @@ -106,14 +101,9 @@ static irqreturn_t ds1287_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction ds1287_irqaction = { - .handler = ds1287_interrupt, - .flags = IRQF_PERCPU | IRQF_TIMER, - .name = "ds1287", -}; - int __init ds1287_clockevent_init(int irq) { + unsigned long flags = IRQF_PERCPU | IRQF_TIMER; struct clock_event_device *cd; cd = &ds1287_clockevent; @@ -121,10 +111,12 @@ int __init ds1287_clockevent_init(int irq) cd->irq = irq; clockevent_set_clock(cd, 32768); cd->max_delta_ns = clockevent_delta2ns(0x7fffffff, cd); + cd->max_delta_ticks = 0x7fffffff; cd->min_delta_ns = clockevent_delta2ns(0x300, cd); + cd->min_delta_ticks = 0x300; cd->cpumask = cpumask_of(0); clockevents_register_device(&ds1287_clockevent); - return setup_irq(irq, &ds1287_irqaction); + return request_irq(irq, ds1287_interrupt, flags, "ds1287", NULL); } diff --git a/arch/mips/kernel/cevt-gic.c b/arch/mips/kernel/cevt-gic.c deleted file mode 100644 index 730eaf92c018..000000000000 --- a/arch/mips/kernel/cevt-gic.c +++ /dev/null @@ -1,104 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2013 Imagination Technologies Ltd. - */ -#include <linux/clockchips.h> -#include <linux/interrupt.h> -#include <linux/percpu.h> -#include <linux/smp.h> -#include <linux/irq.h> - -#include <asm/time.h> -#include <asm/gic.h> -#include <asm/mips-boards/maltaint.h> - -DEFINE_PER_CPU(struct clock_event_device, gic_clockevent_device); -int gic_timer_irq_installed; - - -static int gic_next_event(unsigned long delta, struct clock_event_device *evt) -{ - u64 cnt; - int res; - - cnt = gic_read_count(); - cnt += (u64)delta; - gic_write_compare(cnt); - res = ((int)(gic_read_count() - cnt) >= 0) ? -ETIME : 0; - return res; -} - -void gic_set_clock_mode(enum clock_event_mode mode, - struct clock_event_device *evt) -{ - /* Nothing to do ... */ -} - -irqreturn_t gic_compare_interrupt(int irq, void *dev_id) -{ - struct clock_event_device *cd; - int cpu = smp_processor_id(); - - gic_write_compare(gic_read_compare()); - cd = &per_cpu(gic_clockevent_device, cpu); - cd->event_handler(cd); - return IRQ_HANDLED; -} - -struct irqaction gic_compare_irqaction = { - .handler = gic_compare_interrupt, - .flags = IRQF_PERCPU | IRQF_TIMER, - .name = "timer", -}; - - -void gic_event_handler(struct clock_event_device *dev) -{ -} - -int __cpuinit gic_clockevent_init(void) -{ - unsigned int cpu = smp_processor_id(); - struct clock_event_device *cd; - unsigned int irq; - - if (!cpu_has_counter || !gic_frequency) - return -ENXIO; - - irq = MIPS_GIC_IRQ_BASE; - - cd = &per_cpu(gic_clockevent_device, cpu); - - cd->name = "MIPS GIC"; - cd->features = CLOCK_EVT_FEAT_ONESHOT; - - clockevent_set_clock(cd, gic_frequency); - - /* Calculate the min / max delta */ - cd->max_delta_ns = clockevent_delta2ns(0x7fffffff, cd); - cd->min_delta_ns = clockevent_delta2ns(0x300, cd); - - cd->rating = 300; - cd->irq = irq; - cd->cpumask = cpumask_of(cpu); - cd->set_next_event = gic_next_event; - cd->set_mode = gic_set_clock_mode; - cd->event_handler = gic_event_handler; - - clockevents_register_device(cd); - - GICWRITE(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_MAP), 0x80000002); - GICWRITE(GIC_REG(VPE_LOCAL, GIC_VPE_SMASK), GIC_VPE_SMASK_CMP_MSK); - - if (gic_timer_irq_installed) - return 0; - - gic_timer_irq_installed = 1; - - setup_irq(irq, &gic_compare_irqaction); - irq_set_handler(irq, handle_percpu_irq); - return 0; -} diff --git a/arch/mips/kernel/cevt-gt641xx.c b/arch/mips/kernel/cevt-gt641xx.c index f069460751ab..5b132e8c51da 100644 --- a/arch/mips/kernel/cevt-gt641xx.c +++ b/arch/mips/kernel/cevt-gt641xx.c @@ -1,21 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * GT641xx clockevent routines. * * Copyright (C) 2007 Yoichi Yuasa <yuasa@linux-mips.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include <linux/clockchips.h> #include <linux/init.h> @@ -64,8 +51,7 @@ static int gt641xx_timer0_set_next_event(unsigned long delta, return 0; } -static void gt641xx_timer0_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int gt641xx_timer0_shutdown(struct clock_event_device *evt) { u32 ctrl; @@ -73,21 +59,39 @@ static void gt641xx_timer0_set_mode(enum clock_event_mode mode, ctrl = GT_READ(GT_TC_CONTROL_OFS); ctrl &= ~(GT_TC_CONTROL_ENTC0_MSK | GT_TC_CONTROL_SELTC0_MSK); + GT_WRITE(GT_TC_CONTROL_OFS, ctrl); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - ctrl |= GT_TC_CONTROL_ENTC0_MSK | GT_TC_CONTROL_SELTC0_MSK; - break; - case CLOCK_EVT_MODE_ONESHOT: - ctrl |= GT_TC_CONTROL_ENTC0_MSK; - break; - default: - break; - } + raw_spin_unlock(>641xx_timer_lock); + return 0; +} +static int gt641xx_timer0_set_oneshot(struct clock_event_device *evt) +{ + u32 ctrl; + + raw_spin_lock(>641xx_timer_lock); + + ctrl = GT_READ(GT_TC_CONTROL_OFS); + ctrl &= ~GT_TC_CONTROL_SELTC0_MSK; + ctrl |= GT_TC_CONTROL_ENTC0_MSK; GT_WRITE(GT_TC_CONTROL_OFS, ctrl); raw_spin_unlock(>641xx_timer_lock); + return 0; +} + +static int gt641xx_timer0_set_periodic(struct clock_event_device *evt) +{ + u32 ctrl; + + raw_spin_lock(>641xx_timer_lock); + + ctrl = GT_READ(GT_TC_CONTROL_OFS); + ctrl |= GT_TC_CONTROL_ENTC0_MSK | GT_TC_CONTROL_SELTC0_MSK; + GT_WRITE(GT_TC_CONTROL_OFS, ctrl); + + raw_spin_unlock(>641xx_timer_lock); + return 0; } static void gt641xx_timer0_event_handler(struct clock_event_device *dev) @@ -95,12 +99,16 @@ static void gt641xx_timer0_event_handler(struct clock_event_device *dev) } static struct clock_event_device gt641xx_timer0_clockevent = { - .name = "gt641xx-timer0", - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .irq = GT641XX_TIMER0_IRQ, - .set_next_event = gt641xx_timer0_set_next_event, - .set_mode = gt641xx_timer0_set_mode, - .event_handler = gt641xx_timer0_event_handler, + .name = "gt641xx-timer0", + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .irq = GT641XX_TIMER0_IRQ, + .set_next_event = gt641xx_timer0_set_next_event, + .set_state_shutdown = gt641xx_timer0_shutdown, + .set_state_periodic = gt641xx_timer0_set_periodic, + .set_state_oneshot = gt641xx_timer0_set_oneshot, + .tick_resume = gt641xx_timer0_shutdown, + .event_handler = gt641xx_timer0_event_handler, }; static irqreturn_t gt641xx_timer0_interrupt(int irq, void *dev_id) @@ -112,12 +120,6 @@ static irqreturn_t gt641xx_timer0_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction gt641xx_timer0_irqaction = { - .handler = gt641xx_timer0_interrupt, - .flags = IRQF_PERCPU | IRQF_TIMER, - .name = "gt641xx_timer0", -}; - static int __init gt641xx_timer0_clockevent_init(void) { struct clock_event_device *cd; @@ -131,11 +133,14 @@ static int __init gt641xx_timer0_clockevent_init(void) cd->rating = 200 + gt641xx_base_clock / 10000000; clockevent_set_clock(cd, gt641xx_base_clock); cd->max_delta_ns = clockevent_delta2ns(0x7fffffff, cd); + cd->max_delta_ticks = 0x7fffffff; cd->min_delta_ns = clockevent_delta2ns(0x300, cd); + cd->min_delta_ticks = 0x300; cd->cpumask = cpumask_of(0); clockevents_register_device(>641xx_timer0_clockevent); - return setup_irq(GT641XX_TIMER0_IRQ, >641xx_timer0_irqaction); + return request_irq(GT641XX_TIMER0_IRQ, gt641xx_timer0_interrupt, + IRQF_PERCPU | IRQF_TIMER, "gt641xx_timer0", NULL); } arch_initcall(gt641xx_timer0_clockevent_init); diff --git a/arch/mips/kernel/cevt-r4k.c b/arch/mips/kernel/cevt-r4k.c index 02033eaf8825..5f6e9e2ebbdb 100644 --- a/arch/mips/kernel/cevt-r4k.c +++ b/arch/mips/kernel/cevt-r4k.c @@ -8,21 +8,14 @@ */ #include <linux/clockchips.h> #include <linux/interrupt.h> +#include <linux/cpufreq.h> #include <linux/percpu.h> #include <linux/smp.h> #include <linux/irq.h> -#include <asm/smtc_ipi.h> #include <asm/time.h> #include <asm/cevt-r4k.h> -#include <asm/gic.h> -/* - * The SMTC Kernel for the 34K, 1004K, et. al. replaces several - * of these routines with SMTC-specific variants. - */ - -#ifndef CONFIG_MIPS_MT_SMTC static int mips_next_event(unsigned long delta, struct clock_event_device *evt) { @@ -36,21 +29,107 @@ static int mips_next_event(unsigned long delta, return res; } -#endif /* CONFIG_MIPS_MT_SMTC */ - -void mips_set_clock_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +/** + * calculate_min_delta() - Calculate a good minimum delta for mips_next_event(). + * + * Running under virtualisation can introduce overhead into mips_next_event() in + * the form of hypervisor emulation of CP0_Count/CP0_Compare registers, + * potentially with an unnatural frequency, which makes a fixed min_delta_ns + * value inappropriate as it may be too small. + * + * It can also introduce occasional latency from the guest being descheduled. + * + * This function calculates a good minimum delta based roughly on the 75th + * percentile of the time taken to do the mips_next_event() sequence, in order + * to handle potentially higher overhead while also eliminating outliers due to + * unpredictable hypervisor latency (which can be handled by retries). + * + * Return: An appropriate minimum delta for the clock event device. + */ +static unsigned int calculate_min_delta(void) { - /* Nothing to do ... */ + unsigned int cnt, i, j, k, l; + unsigned int buf1[4], buf2[3]; + unsigned int min_delta; + + /* + * Calculate the median of 5 75th percentiles of 5 samples of how long + * it takes to set CP0_Compare = CP0_Count + delta. + */ + for (i = 0; i < 5; ++i) { + for (j = 0; j < 5; ++j) { + /* + * This is like the code in mips_next_event(), and + * directly measures the borderline "safe" delta. + */ + cnt = read_c0_count(); + write_c0_compare(cnt); + cnt = read_c0_count() - cnt; + + /* Sorted insert into buf1 */ + for (k = 0; k < j; ++k) { + if (cnt < buf1[k]) { + l = min_t(unsigned int, + j, ARRAY_SIZE(buf1) - 1); + for (; l > k; --l) + buf1[l] = buf1[l - 1]; + break; + } + } + if (k < ARRAY_SIZE(buf1)) + buf1[k] = cnt; + } + + /* Sorted insert of 75th percentile into buf2 */ + for (k = 0; k < i && k < ARRAY_SIZE(buf2); ++k) { + if (buf1[ARRAY_SIZE(buf1) - 1] < buf2[k]) { + l = min_t(unsigned int, + i, ARRAY_SIZE(buf2) - 1); + for (; l > k; --l) + buf2[l] = buf2[l - 1]; + break; + } + } + if (k < ARRAY_SIZE(buf2)) + buf2[k] = buf1[ARRAY_SIZE(buf1) - 1]; + } + + /* Use 2 * median of 75th percentiles */ + min_delta = buf2[ARRAY_SIZE(buf2) - 1] * 2; + + /* Don't go too low */ + if (min_delta < 0x300) + min_delta = 0x300; + + pr_debug("%s: median 75th percentile=%#x, min_delta=%#x\n", + __func__, buf2[ARRAY_SIZE(buf2) - 1], min_delta); + return min_delta; } DEFINE_PER_CPU(struct clock_event_device, mips_clockevent_device); int cp0_timer_irq_installed; -#ifndef CONFIG_MIPS_MT_SMTC +/* + * Possibly handle a performance counter interrupt. + * Return true if the timer interrupt should not be checked + */ +static inline int handle_perf_irq(int r2) +{ + /* + * The performance counter overflow interrupt may be shared with the + * timer interrupt (cp0_perfcount_irq < 0). If it is and a + * performance counter has overflowed (perf_irq() == IRQ_HANDLED) + * and we can't reliably determine if a counter interrupt has also + * happened (!r2) then don't check for a timer interrupt. + */ + return (cp0_perfcount_irq < 0) && + perf_irq() == IRQ_HANDLED && + !r2; +} + irqreturn_t c0_compare_interrupt(int irq, void *dev_id) { - const int r2 = cpu_has_mips_r2; + const int r2 = cpu_has_mips_r2_r6; struct clock_event_device *cd; int cpu = smp_processor_id(); @@ -61,32 +140,32 @@ irqreturn_t c0_compare_interrupt(int irq, void *dev_id) * the performance counter interrupt handler anyway. */ if (handle_perf_irq(r2)) - goto out; + return IRQ_HANDLED; /* * The same applies to performance counter interrupts. But with the * above we now know that the reason we got here must be a timer * interrupt. Being the paranoiacs we are we check anyway. */ - if (!r2 || (read_c0_cause() & (1 << 30))) { + if (!r2 || (read_c0_cause() & CAUSEF_TI)) { /* Clear Count/Compare Interrupt */ write_c0_compare(read_c0_compare()); cd = &per_cpu(mips_clockevent_device, cpu); -#ifdef CONFIG_CEVT_GIC - if (!gic_present) -#endif cd->event_handler(cd); + + return IRQ_HANDLED; } -out: - return IRQ_HANDLED; + return IRQ_NONE; } -#endif /* Not CONFIG_MIPS_MT_SMTC */ - struct irqaction c0_compare_irqaction = { .handler = c0_compare_interrupt, - .flags = IRQF_PERCPU | IRQF_TIMER, + /* + * IRQF_SHARED: The timer interrupt may be shared with other interrupts + * such as perf counter and FDC interrupts. + */ + .flags = IRQF_PERCPU | IRQF_TIMER | IRQF_SHARED, .name = "timer", }; @@ -100,10 +179,7 @@ void mips_event_handler(struct clock_event_device *dev) */ static int c0_compare_int_pending(void) { -#ifdef CONFIG_IRQ_GIC - if (cpu_has_veic) - return gic_get_timer_pending(); -#endif + /* When cpu_has_mips_r2, this checks Cause.TI instead of Cause.IP7 */ return (read_c0_cause() >> cp0_compare_irq_shift) & (1ul << CAUSEB_IP); } @@ -119,16 +195,12 @@ int c0_compare_int_usable(void) unsigned int delta; unsigned int cnt; -#ifdef CONFIG_KVM_GUEST - return 1; -#endif - /* * IP7 already pending? Try to clear it by acking the timer. */ if (c0_compare_int_pending()) { cnt = read_c0_count(); - write_c0_compare(cnt); + write_c0_compare(cnt - 1); back_to_back_c0_hazard(); while (read_c0_count() < (cnt + COMPARE_INT_SEEN_TICKS)) if (!c0_compare_int_pending()) @@ -156,7 +228,7 @@ int c0_compare_int_usable(void) if (!c0_compare_int_pending()) return 0; cnt = read_c0_count(); - write_c0_compare(cnt); + write_c0_compare(cnt - 1); back_to_back_c0_hazard(); while (read_c0_count() < (cnt + COMPARE_INT_SEEN_TICKS)) if (!c0_compare_int_pending()) @@ -170,12 +242,60 @@ int c0_compare_int_usable(void) return 1; } -#ifndef CONFIG_MIPS_MT_SMTC -int __cpuinit r4k_clockevent_init(void) +unsigned int __weak get_c0_compare_int(void) { + return MIPS_CPU_IRQ_BASE + cp0_compare_irq; +} + +#ifdef CONFIG_CPU_FREQ + +static unsigned long mips_ref_freq; + +static int r4k_cpufreq_callback(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_freqs *freq = data; + struct clock_event_device *cd; + unsigned long rate; + int cpu; + + if (!mips_ref_freq) + mips_ref_freq = freq->old; + + if (val == CPUFREQ_POSTCHANGE) { + rate = cpufreq_scale(mips_hpt_frequency, mips_ref_freq, + freq->new); + + for_each_cpu(cpu, freq->policy->cpus) { + cd = &per_cpu(mips_clockevent_device, cpu); + + clockevents_update_freq(cd, rate); + } + } + + return 0; +} + +static struct notifier_block r4k_cpufreq_notifier = { + .notifier_call = r4k_cpufreq_callback, +}; + +static int __init r4k_register_cpufreq_notifier(void) +{ + return cpufreq_register_notifier(&r4k_cpufreq_notifier, + CPUFREQ_TRANSITION_NOTIFIER); + +} +core_initcall(r4k_register_cpufreq_notifier); + +#endif /* !CONFIG_CPU_FREQ */ + +int r4k_clockevent_init(void) +{ + unsigned long flags = IRQF_PERCPU | IRQF_TIMER | IRQF_SHARED; unsigned int cpu = smp_processor_id(); struct clock_event_device *cd; - unsigned int irq; + unsigned int irq, min_delta; if (!cpu_has_counter || !mips_hpt_frequency) return -ENXIO; @@ -183,46 +303,38 @@ int __cpuinit r4k_clockevent_init(void) if (!c0_compare_int_usable()) return -ENXIO; - /* - * With vectored interrupts things are getting platform specific. - * get_c0_compare_int is a hook to allow a platform to return the - * interrupt number of it's liking. - */ - irq = MIPS_CPU_IRQ_BASE + cp0_compare_irq; - if (get_c0_compare_int) - irq = get_c0_compare_int(); - cd = &per_cpu(mips_clockevent_device, cpu); cd->name = "MIPS"; - cd->features = CLOCK_EVT_FEAT_ONESHOT; + cd->features = CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_C3STOP | + CLOCK_EVT_FEAT_PERCPU; - clockevent_set_clock(cd, mips_hpt_frequency); - - /* Calculate the min / max delta */ - cd->max_delta_ns = clockevent_delta2ns(0x7fffffff, cd); - cd->min_delta_ns = clockevent_delta2ns(0x300, cd); + min_delta = calculate_min_delta(); cd->rating = 300; - cd->irq = irq; cd->cpumask = cpumask_of(cpu); cd->set_next_event = mips_next_event; - cd->set_mode = mips_set_clock_mode; cd->event_handler = mips_event_handler; -#ifdef CONFIG_CEVT_GIC - if (!gic_present) -#endif - clockevents_register_device(cd); + clockevents_config_and_register(cd, mips_hpt_frequency, min_delta, 0x7fffffff); if (cp0_timer_irq_installed) return 0; cp0_timer_irq_installed = 1; - setup_irq(irq, &c0_compare_irqaction); + /* + * With vectored interrupts things are getting platform specific. + * get_c0_compare_int is a hook to allow a platform to return the + * interrupt number of its liking. + */ + irq = get_c0_compare_int(); + + if (request_irq(irq, c0_compare_interrupt, flags, "timer", + c0_compare_interrupt)) + pr_err("Failed to request irq %d (timer)\n", irq); return 0; } -#endif /* Not CONFIG_MIPS_MT_SMTC */ diff --git a/arch/mips/kernel/cevt-sb1250.c b/arch/mips/kernel/cevt-sb1250.c index 200f2778bf36..0451273fab9f 100644 --- a/arch/mips/kernel/cevt-sb1250.c +++ b/arch/mips/kernel/cevt-sb1250.c @@ -1,19 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2000, 2001 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ #include <linux/clockchips.h> #include <linux/interrupt.h> @@ -38,8 +25,20 @@ * The general purpose timer ticks at 1MHz independent if * the rest of the system */ -static void sibyte_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) + +static int sibyte_shutdown(struct clock_event_device *evt) +{ + void __iomem *cfg; + + cfg = IOADDR(A_SCD_TIMER_REGISTER(smp_processor_id(), R_SCD_TIMER_CFG)); + + /* Stop the timer until we actually program a shot */ + __raw_writeq(0, cfg); + + return 0; +} + +static int sibyte_set_periodic(struct clock_event_device *evt) { unsigned int cpu = smp_processor_id(); void __iomem *cfg, *init; @@ -47,24 +46,11 @@ static void sibyte_set_mode(enum clock_event_mode mode, cfg = IOADDR(A_SCD_TIMER_REGISTER(cpu, R_SCD_TIMER_CFG)); init = IOADDR(A_SCD_TIMER_REGISTER(cpu, R_SCD_TIMER_INIT)); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - __raw_writeq(0, cfg); - __raw_writeq((V_SCD_TIMER_FREQ / HZ) - 1, init); - __raw_writeq(M_SCD_TIMER_ENABLE | M_SCD_TIMER_MODE_CONTINUOUS, - cfg); - break; - - case CLOCK_EVT_MODE_ONESHOT: - /* Stop the timer until we actually program a shot */ - case CLOCK_EVT_MODE_SHUTDOWN: - __raw_writeq(0, cfg); - break; - - case CLOCK_EVT_MODE_UNUSED: /* shuddup gcc */ - case CLOCK_EVT_MODE_RESUME: - ; - } + __raw_writeq(0, cfg); + __raw_writeq((V_SCD_TIMER_FREQ / HZ) - 1, init); + __raw_writeq(M_SCD_TIMER_ENABLE | M_SCD_TIMER_MODE_CONTINUOUS, cfg); + + return 0; } static int sibyte_next_event(unsigned long delta, struct clock_event_device *cd) @@ -89,7 +75,7 @@ static irqreturn_t sibyte_counter_handler(int irq, void *dev_id) void __iomem *cfg; unsigned long tmode; - if (cd->mode == CLOCK_EVT_MODE_PERIODIC) + if (clockevent_state_periodic(cd)) tmode = M_SCD_TIMER_ENABLE | M_SCD_TIMER_MODE_CONTINUOUS; else tmode = 0; @@ -104,16 +90,15 @@ static irqreturn_t sibyte_counter_handler(int irq, void *dev_id) } static DEFINE_PER_CPU(struct clock_event_device, sibyte_hpt_clockevent); -static DEFINE_PER_CPU(struct irqaction, sibyte_hpt_irqaction); static DEFINE_PER_CPU(char [18], sibyte_hpt_name); -void __cpuinit sb1250_clockevent_init(void) +void sb1250_clockevent_init(void) { unsigned int cpu = smp_processor_id(); unsigned int irq = K_INT_TIMER_0 + cpu; - struct irqaction *action = &per_cpu(sibyte_hpt_irqaction, cpu); struct clock_event_device *cd = &per_cpu(sibyte_hpt_clockevent, cpu); unsigned char *name = per_cpu(sibyte_hpt_name, cpu); + unsigned long flags = IRQF_PERCPU | IRQF_TIMER; /* Only have 4 general purpose timers, and we use last one as hpt */ BUG_ON(cpu > 2); @@ -124,12 +109,16 @@ void __cpuinit sb1250_clockevent_init(void) CLOCK_EVT_FEAT_ONESHOT; clockevent_set_clock(cd, V_SCD_TIMER_FREQ); cd->max_delta_ns = clockevent_delta2ns(0x7fffff, cd); + cd->max_delta_ticks = 0x7fffff; cd->min_delta_ns = clockevent_delta2ns(2, cd); + cd->min_delta_ticks = 2; cd->rating = 200; cd->irq = irq; cd->cpumask = cpumask_of(cpu); cd->set_next_event = sibyte_next_event; - cd->set_mode = sibyte_set_mode; + cd->set_state_shutdown = sibyte_shutdown; + cd->set_state_periodic = sibyte_set_periodic; + cd->set_state_oneshot = sibyte_shutdown; clockevents_register_device(cd); sb1250_mask_irq(cpu, irq); @@ -143,11 +132,7 @@ void __cpuinit sb1250_clockevent_init(void) sb1250_unmask_irq(cpu, irq); - action->handler = sibyte_counter_handler; - action->flags = IRQF_PERCPU | IRQF_TIMER; - action->name = name; - action->dev_id = cd; - irq_set_affinity(irq, cpumask_of(cpu)); - setup_irq(irq, action); + if (request_irq(irq, sibyte_counter_handler, flags, name, cd)) + pr_err("Failed to request irq %d (%s)\n", irq, name); } diff --git a/arch/mips/kernel/cevt-smtc.c b/arch/mips/kernel/cevt-smtc.c deleted file mode 100644 index 9de5ed7ef1a3..000000000000 --- a/arch/mips/kernel/cevt-smtc.c +++ /dev/null @@ -1,324 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2007 MIPS Technologies, Inc. - * Copyright (C) 2007 Ralf Baechle <ralf@linux-mips.org> - * Copyright (C) 2008 Kevin D. Kissell, Paralogos sarl - */ -#include <linux/clockchips.h> -#include <linux/interrupt.h> -#include <linux/percpu.h> -#include <linux/smp.h> -#include <linux/irq.h> - -#include <asm/smtc_ipi.h> -#include <asm/time.h> -#include <asm/cevt-r4k.h> - -/* - * Variant clock event timer support for SMTC on MIPS 34K, 1004K - * or other MIPS MT cores. - * - * Notes on SMTC Support: - * - * SMTC has multiple microthread TCs pretending to be Linux CPUs. - * But there's only one Count/Compare pair per VPE, and Compare - * interrupts are taken opportunisitically by available TCs - * bound to the VPE with the Count register. The new timer - * framework provides for global broadcasts, but we really - * want VPE-level multicasts for best behavior. So instead - * of invoking the high-level clock-event broadcast code, - * this version of SMTC support uses the historical SMTC - * multicast mechanisms "under the hood", appearing to the - * generic clock layer as if the interrupts are per-CPU. - * - * The approach taken here is to maintain a set of NR_CPUS - * virtual timers, and track which "CPU" needs to be alerted - * at each event. - * - * It's unlikely that we'll see a MIPS MT core with more than - * 2 VPEs, but we *know* that we won't need to handle more - * VPEs than we have "CPUs". So NCPUs arrays of NCPUs elements - * is always going to be overkill, but always going to be enough. - */ - -unsigned long smtc_nexttime[NR_CPUS][NR_CPUS]; -static int smtc_nextinvpe[NR_CPUS]; - -/* - * Timestamps stored are absolute values to be programmed - * into Count register. Valid timestamps will never be zero. - * If a Zero Count value is actually calculated, it is converted - * to be a 1, which will introduce 1 or two CPU cycles of error - * roughly once every four billion events, which at 1000 HZ means - * about once every 50 days. If that's actually a problem, one - * could alternate squashing 0 to 1 and to -1. - */ - -#define MAKEVALID(x) (((x) == 0L) ? 1L : (x)) -#define ISVALID(x) ((x) != 0L) - -/* - * Time comparison is subtle, as it's really truncated - * modular arithmetic. - */ - -#define IS_SOONER(a, b, reference) \ - (((a) - (unsigned long)(reference)) < ((b) - (unsigned long)(reference))) - -/* - * CATCHUP_INCREMENT, used when the function falls behind the counter. - * Could be an increasing function instead of a constant; - */ - -#define CATCHUP_INCREMENT 64 - -static int mips_next_event(unsigned long delta, - struct clock_event_device *evt) -{ - unsigned long flags; - unsigned int mtflags; - unsigned long timestamp, reference, previous; - unsigned long nextcomp = 0L; - int vpe = current_cpu_data.vpe_id; - int cpu = smp_processor_id(); - local_irq_save(flags); - mtflags = dmt(); - - /* - * Maintain the per-TC virtual timer - * and program the per-VPE shared Count register - * as appropriate here... - */ - reference = (unsigned long)read_c0_count(); - timestamp = MAKEVALID(reference + delta); - /* - * To really model the clock, we have to catch the case - * where the current next-in-VPE timestamp is the old - * timestamp for the calling CPE, but the new value is - * in fact later. In that case, we have to do a full - * scan and discover the new next-in-VPE CPU id and - * timestamp. - */ - previous = smtc_nexttime[vpe][cpu]; - if (cpu == smtc_nextinvpe[vpe] && ISVALID(previous) - && IS_SOONER(previous, timestamp, reference)) { - int i; - int soonest = cpu; - - /* - * Update timestamp array here, so that new - * value gets considered along with those of - * other virtual CPUs on the VPE. - */ - smtc_nexttime[vpe][cpu] = timestamp; - for_each_online_cpu(i) { - if (ISVALID(smtc_nexttime[vpe][i]) - && IS_SOONER(smtc_nexttime[vpe][i], - smtc_nexttime[vpe][soonest], reference)) { - soonest = i; - } - } - smtc_nextinvpe[vpe] = soonest; - nextcomp = smtc_nexttime[vpe][soonest]; - /* - * Otherwise, we don't have to process the whole array rank, - * we just have to see if the event horizon has gotten closer. - */ - } else { - if (!ISVALID(smtc_nexttime[vpe][smtc_nextinvpe[vpe]]) || - IS_SOONER(timestamp, - smtc_nexttime[vpe][smtc_nextinvpe[vpe]], reference)) { - smtc_nextinvpe[vpe] = cpu; - nextcomp = timestamp; - } - /* - * Since next-in-VPE may me the same as the executing - * virtual CPU, we update the array *after* checking - * its value. - */ - smtc_nexttime[vpe][cpu] = timestamp; - } - - /* - * It may be that, in fact, we don't need to update Compare, - * but if we do, we want to make sure we didn't fall into - * a crack just behind Count. - */ - if (ISVALID(nextcomp)) { - write_c0_compare(nextcomp); - ehb(); - /* - * We never return an error, we just make sure - * that we trigger the handlers as quickly as - * we can if we fell behind. - */ - while ((nextcomp - (unsigned long)read_c0_count()) - > (unsigned long)LONG_MAX) { - nextcomp += CATCHUP_INCREMENT; - write_c0_compare(nextcomp); - ehb(); - } - } - emt(mtflags); - local_irq_restore(flags); - return 0; -} - - -void smtc_distribute_timer(int vpe) -{ - unsigned long flags; - unsigned int mtflags; - int cpu; - struct clock_event_device *cd; - unsigned long nextstamp; - unsigned long reference; - - -repeat: - nextstamp = 0L; - for_each_online_cpu(cpu) { - /* - * Find virtual CPUs within the current VPE who have - * unserviced timer requests whose time is now past. - */ - local_irq_save(flags); - mtflags = dmt(); - if (cpu_data[cpu].vpe_id == vpe && - ISVALID(smtc_nexttime[vpe][cpu])) { - reference = (unsigned long)read_c0_count(); - if ((smtc_nexttime[vpe][cpu] - reference) - > (unsigned long)LONG_MAX) { - smtc_nexttime[vpe][cpu] = 0L; - emt(mtflags); - local_irq_restore(flags); - /* - * We don't send IPIs to ourself. - */ - if (cpu != smp_processor_id()) { - smtc_send_ipi(cpu, SMTC_CLOCK_TICK, 0); - } else { - cd = &per_cpu(mips_clockevent_device, cpu); - cd->event_handler(cd); - } - } else { - /* Local to VPE but Valid Time not yet reached. */ - if (!ISVALID(nextstamp) || - IS_SOONER(smtc_nexttime[vpe][cpu], nextstamp, - reference)) { - smtc_nextinvpe[vpe] = cpu; - nextstamp = smtc_nexttime[vpe][cpu]; - } - emt(mtflags); - local_irq_restore(flags); - } - } else { - emt(mtflags); - local_irq_restore(flags); - - } - } - /* Reprogram for interrupt at next soonest timestamp for VPE */ - if (ISVALID(nextstamp)) { - write_c0_compare(nextstamp); - ehb(); - if ((nextstamp - (unsigned long)read_c0_count()) - > (unsigned long)LONG_MAX) - goto repeat; - } -} - - -irqreturn_t c0_compare_interrupt(int irq, void *dev_id) -{ - int cpu = smp_processor_id(); - - /* If we're running SMTC, we've got MIPS MT and therefore MIPS32R2 */ - handle_perf_irq(1); - - if (read_c0_cause() & (1 << 30)) { - /* Clear Count/Compare Interrupt */ - write_c0_compare(read_c0_compare()); - smtc_distribute_timer(cpu_data[cpu].vpe_id); - } - return IRQ_HANDLED; -} - - -int __cpuinit smtc_clockevent_init(void) -{ - uint64_t mips_freq = mips_hpt_frequency; - unsigned int cpu = smp_processor_id(); - struct clock_event_device *cd; - unsigned int irq; - int i; - int j; - - if (!cpu_has_counter || !mips_hpt_frequency) - return -ENXIO; - if (cpu == 0) { - for (i = 0; i < num_possible_cpus(); i++) { - smtc_nextinvpe[i] = 0; - for (j = 0; j < num_possible_cpus(); j++) - smtc_nexttime[i][j] = 0L; - } - /* - * SMTC also can't have the usablility test - * run by secondary TCs once Compare is in use. - */ - if (!c0_compare_int_usable()) - return -ENXIO; - } - - /* - * With vectored interrupts things are getting platform specific. - * get_c0_compare_int is a hook to allow a platform to return the - * interrupt number of it's liking. - */ - irq = MIPS_CPU_IRQ_BASE + cp0_compare_irq; - if (get_c0_compare_int) - irq = get_c0_compare_int(); - - cd = &per_cpu(mips_clockevent_device, cpu); - - cd->name = "MIPS"; - cd->features = CLOCK_EVT_FEAT_ONESHOT; - - /* Calculate the min / max delta */ - cd->mult = div_sc((unsigned long) mips_freq, NSEC_PER_SEC, 32); - cd->shift = 32; - cd->max_delta_ns = clockevent_delta2ns(0x7fffffff, cd); - cd->min_delta_ns = clockevent_delta2ns(0x300, cd); - - cd->rating = 300; - cd->irq = irq; - cd->cpumask = cpumask_of(cpu); - cd->set_next_event = mips_next_event; - cd->set_mode = mips_set_clock_mode; - cd->event_handler = mips_event_handler; - - clockevents_register_device(cd); - - /* - * On SMTC we only want to do the data structure - * initialization and IRQ setup once. - */ - if (cpu) - return 0; - /* - * And we need the hwmask associated with the c0_compare - * vector to be initialized. - */ - irq_hwmask[irq] = (0x100 << cp0_compare_irq); - if (cp0_timer_irq_installed) - return 0; - - cp0_timer_irq_installed = 1; - - setup_irq(irq, &c0_compare_irqaction); - - return 0; -} diff --git a/arch/mips/kernel/cevt-txx9.c b/arch/mips/kernel/cevt-txx9.c index 2ae08462e46e..d761ead2e7fe 100644 --- a/arch/mips/kernel/cevt-txx9.c +++ b/arch/mips/kernel/cevt-txx9.c @@ -14,6 +14,7 @@ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/irq.h> +#include <linux/sched_clock.h> #include <asm/time.h> #include <asm/txx9tmr.h> @@ -26,7 +27,7 @@ struct txx9_clocksource { struct txx9_tmr_reg __iomem *tmrptr; }; -static cycle_t txx9_cs_read(struct clocksource *cs) +static u64 txx9_cs_read(struct clocksource *cs) { struct txx9_clocksource *txx9_cs = container_of(cs, struct txx9_clocksource, cs); @@ -46,6 +47,11 @@ static struct txx9_clocksource txx9_clocksource = { }, }; +static u64 notrace txx9_read_sched_clock(void) +{ + return __raw_readl(&txx9_clocksource.tmrptr->trr); +} + void __init txx9_clocksource_init(unsigned long baseaddr, unsigned int imbusclk) { @@ -61,6 +67,9 @@ void __init txx9_clocksource_init(unsigned long baseaddr, __raw_writel(1 << TXX9_CLOCKSOURCE_BITS, &tmrptr->cpra); __raw_writel(TCR_BASE | TXx9_TMTCR_TCE, &tmrptr->tcr); txx9_clocksource.tmrptr = tmrptr; + + sched_clock_register(txx9_read_sched_clock, TXX9_CLOCKSOURCE_BITS, + TIMER_CLK(imbusclk)); } struct txx9_clock_event_device { @@ -76,36 +85,54 @@ static void txx9tmr_stop_and_clear(struct txx9_tmr_reg __iomem *tmrptr) __raw_writel(0, &tmrptr->tisr); } -static void txx9tmr_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int txx9tmr_set_state_periodic(struct clock_event_device *evt) +{ + struct txx9_clock_event_device *txx9_cd = + container_of(evt, struct txx9_clock_event_device, cd); + struct txx9_tmr_reg __iomem *tmrptr = txx9_cd->tmrptr; + + txx9tmr_stop_and_clear(tmrptr); + + __raw_writel(TXx9_TMITMR_TIIE | TXx9_TMITMR_TZCE, &tmrptr->itmr); + /* start timer */ + __raw_writel(((u64)(NSEC_PER_SEC / HZ) * evt->mult) >> evt->shift, + &tmrptr->cpra); + __raw_writel(TCR_BASE | TXx9_TMTCR_TCE, &tmrptr->tcr); + return 0; +} + +static int txx9tmr_set_state_oneshot(struct clock_event_device *evt) +{ + struct txx9_clock_event_device *txx9_cd = + container_of(evt, struct txx9_clock_event_device, cd); + struct txx9_tmr_reg __iomem *tmrptr = txx9_cd->tmrptr; + + txx9tmr_stop_and_clear(tmrptr); + __raw_writel(TXx9_TMITMR_TIIE, &tmrptr->itmr); + return 0; +} + +static int txx9tmr_set_state_shutdown(struct clock_event_device *evt) { struct txx9_clock_event_device *txx9_cd = container_of(evt, struct txx9_clock_event_device, cd); struct txx9_tmr_reg __iomem *tmrptr = txx9_cd->tmrptr; txx9tmr_stop_and_clear(tmrptr); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - __raw_writel(TXx9_TMITMR_TIIE | TXx9_TMITMR_TZCE, - &tmrptr->itmr); - /* start timer */ - __raw_writel(((u64)(NSEC_PER_SEC / HZ) * evt->mult) >> - evt->shift, - &tmrptr->cpra); - __raw_writel(TCR_BASE | TXx9_TMTCR_TCE, &tmrptr->tcr); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - __raw_writel(0, &tmrptr->itmr); - break; - case CLOCK_EVT_MODE_ONESHOT: - __raw_writel(TXx9_TMITMR_TIIE, &tmrptr->itmr); - break; - case CLOCK_EVT_MODE_RESUME: - __raw_writel(TIMER_CCD, &tmrptr->ccdr); - __raw_writel(0, &tmrptr->itmr); - break; - } + __raw_writel(0, &tmrptr->itmr); + return 0; +} + +static int txx9tmr_tick_resume(struct clock_event_device *evt) +{ + struct txx9_clock_event_device *txx9_cd = + container_of(evt, struct txx9_clock_event_device, cd); + struct txx9_tmr_reg __iomem *tmrptr = txx9_cd->tmrptr; + + txx9tmr_stop_and_clear(tmrptr); + __raw_writel(TIMER_CCD, &tmrptr->ccdr); + __raw_writel(0, &tmrptr->itmr); + return 0; } static int txx9tmr_set_next_event(unsigned long delta, @@ -124,12 +151,15 @@ static int txx9tmr_set_next_event(unsigned long delta, static struct txx9_clock_event_device txx9_clock_event_device = { .cd = { - .name = "TXx9", - .features = CLOCK_EVT_FEAT_PERIODIC | - CLOCK_EVT_FEAT_ONESHOT, - .rating = 200, - .set_mode = txx9tmr_set_mode, - .set_next_event = txx9tmr_set_next_event, + .name = "TXx9", + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .rating = 200, + .set_state_shutdown = txx9tmr_set_state_shutdown, + .set_state_periodic = txx9tmr_set_state_periodic, + .set_state_oneshot = txx9tmr_set_state_oneshot, + .tick_resume = txx9tmr_tick_resume, + .set_next_event = txx9tmr_set_next_event, }, }; @@ -144,13 +174,6 @@ static irqreturn_t txx9tmr_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction txx9tmr_irq = { - .handler = txx9tmr_interrupt, - .flags = IRQF_PERCPU | IRQF_TIMER, - .name = "txx9tmr", - .dev_id = &txx9_clock_event_device, -}; - void __init txx9_clockevent_init(unsigned long baseaddr, int irq, unsigned int imbusclk) { @@ -166,11 +189,15 @@ void __init txx9_clockevent_init(unsigned long baseaddr, int irq, clockevent_set_clock(cd, TIMER_CLK(imbusclk)); cd->max_delta_ns = clockevent_delta2ns(0xffffffff >> (32 - TXX9_TIMER_BITS), cd); + cd->max_delta_ticks = 0xffffffff >> (32 - TXX9_TIMER_BITS); cd->min_delta_ns = clockevent_delta2ns(0xf, cd); + cd->min_delta_ticks = 0xf; cd->irq = irq; - cd->cpumask = cpumask_of(0), + cd->cpumask = cpumask_of(0); clockevents_register_device(cd); - setup_irq(irq, &txx9tmr_irq); + if (request_irq(irq, txx9tmr_interrupt, IRQF_PERCPU | IRQF_TIMER, + "txx9tmr", &txx9_clock_event_device)) + pr_err("Failed to request irq %d (txx9tmr)\n", irq); printk(KERN_INFO "TXx9: clockevent device at 0x%lx, irq %d\n", baseaddr, irq); } diff --git a/arch/mips/kernel/cmpxchg.c b/arch/mips/kernel/cmpxchg.c new file mode 100644 index 000000000000..c371def2302d --- /dev/null +++ b/arch/mips/kernel/cmpxchg.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017 Imagination Technologies + * Author: Paul Burton <paul.burton@mips.com> + */ + +#include <linux/bitops.h> +#include <asm/cmpxchg.h> + +unsigned long __xchg_small(volatile void *ptr, unsigned long val, unsigned int size) +{ + u32 old32, new32, load32, mask; + volatile u32 *ptr32; + unsigned int shift; + + /* Check that ptr is naturally aligned */ + WARN_ON((unsigned long)ptr & (size - 1)); + + /* Mask value to the correct size. */ + mask = GENMASK((size * BITS_PER_BYTE) - 1, 0); + val &= mask; + + /* + * Calculate a shift & mask that correspond to the value we wish to + * exchange within the naturally aligned 4 byte integer that includes + * it. + */ + shift = (unsigned long)ptr & 0x3; + if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) + shift ^= sizeof(u32) - size; + shift *= BITS_PER_BYTE; + mask <<= shift; + + /* + * Calculate a pointer to the naturally aligned 4 byte integer that + * includes our byte of interest, and load its value. + */ + ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3); + load32 = *ptr32; + + do { + old32 = load32; + new32 = (load32 & ~mask) | (val << shift); + load32 = arch_cmpxchg(ptr32, old32, new32); + } while (load32 != old32); + + return (load32 & mask) >> shift; +} + +unsigned long __cmpxchg_small(volatile void *ptr, unsigned long old, + unsigned long new, unsigned int size) +{ + u32 mask, old32, new32, load32, load; + volatile u32 *ptr32; + unsigned int shift; + + /* Check that ptr is naturally aligned */ + WARN_ON((unsigned long)ptr & (size - 1)); + + /* Mask inputs to the correct size. */ + mask = GENMASK((size * BITS_PER_BYTE) - 1, 0); + old &= mask; + new &= mask; + + /* + * Calculate a shift & mask that correspond to the value we wish to + * compare & exchange within the naturally aligned 4 byte integer + * that includes it. + */ + shift = (unsigned long)ptr & 0x3; + if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) + shift ^= sizeof(u32) - size; + shift *= BITS_PER_BYTE; + mask <<= shift; + + /* + * Calculate a pointer to the naturally aligned 4 byte integer that + * includes our byte of interest, and load its value. + */ + ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3); + load32 = *ptr32; + + while (true) { + /* + * Ensure the byte we want to exchange matches the expected + * old value, and if not then bail. + */ + load = (load32 & mask) >> shift; + if (load != old) + return load; + + /* + * Calculate the old & new values of the naturally aligned + * 4 byte integer that include the byte we want to exchange. + * Attempt to exchange the old value for the new value, and + * return if we succeed. + */ + old32 = (load32 & ~mask) | (old << shift); + new32 = (load32 & ~mask) | (new << shift); + load32 = arch_cmpxchg(ptr32, old32, new32); + if (load32 == old32) + return old; + } +} +EXPORT_SYMBOL(__cmpxchg_small); diff --git a/arch/mips/kernel/cps-vec-ns16550.S b/arch/mips/kernel/cps-vec-ns16550.S new file mode 100644 index 000000000000..30725e1df987 --- /dev/null +++ b/arch/mips/kernel/cps-vec-ns16550.S @@ -0,0 +1,212 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2015 Imagination Technologies + * Author: Paul Burton <paul.burton@mips.com> + */ + +#include <asm/addrspace.h> +#include <asm/asm.h> +#include <asm/asm-offsets.h> +#include <asm/mipsregs.h> +#include <asm/regdef.h> +#include <linux/serial_reg.h> + +#define UART_TX_OFS (UART_TX << CONFIG_MIPS_CPS_NS16550_SHIFT) +#define UART_LSR_OFS (UART_LSR << CONFIG_MIPS_CPS_NS16550_SHIFT) + +#if CONFIG_MIPS_CPS_NS16550_WIDTH == 1 +# define UART_L lb +# define UART_S sb +#elif CONFIG_MIPS_CPS_NS16550_WIDTH == 2 +# define UART_L lh +# define UART_S sh +#elif CONFIG_MIPS_CPS_NS16550_WIDTH == 4 +# define UART_L lw +# define UART_S sw +#else +# define UART_L lb +# define UART_S sb +#endif + +/** + * _mips_cps_putc() - write a character to the UART + * @a0: ASCII character to write + * @t9: UART base address + */ +LEAF(_mips_cps_putc) +1: UART_L t0, UART_LSR_OFS(t9) + andi t0, t0, UART_LSR_TEMT + beqz t0, 1b + UART_S a0, UART_TX_OFS(t9) + jr ra + END(_mips_cps_putc) + +/** + * _mips_cps_puts() - write a string to the UART + * @a0: pointer to NULL-terminated ASCII string + * @t9: UART base address + * + * Write a null-terminated ASCII string to the UART. + */ +NESTED(_mips_cps_puts, 0, ra) + move s7, ra + move s6, a0 + +1: lb a0, 0(s6) + beqz a0, 2f + jal _mips_cps_putc + PTR_ADDIU s6, s6, 1 + b 1b + +2: jr s7 + END(_mips_cps_puts) + +/** + * _mips_cps_putx4 - write a 4b hex value to the UART + * @a0: the 4b value to write to the UART + * @t9: UART base address + * + * Write a single hexadecimal character to the UART. + */ +NESTED(_mips_cps_putx4, 0, ra) + andi a0, a0, 0xf + li t0, '0' + blt a0, 10, 1f + li t0, 'a' + addiu a0, a0, -10 +1: addu a0, a0, t0 + b _mips_cps_putc + END(_mips_cps_putx4) + +/** + * _mips_cps_putx8 - write an 8b hex value to the UART + * @a0: the 8b value to write to the UART + * @t9: UART base address + * + * Write an 8 bit value (ie. 2 hexadecimal characters) to the UART. + */ +NESTED(_mips_cps_putx8, 0, ra) + move s3, ra + move s2, a0 + srl a0, a0, 4 + jal _mips_cps_putx4 + move a0, s2 + move ra, s3 + b _mips_cps_putx4 + END(_mips_cps_putx8) + +/** + * _mips_cps_putx16 - write a 16b hex value to the UART + * @a0: the 16b value to write to the UART + * @t9: UART base address + * + * Write a 16 bit value (ie. 4 hexadecimal characters) to the UART. + */ +NESTED(_mips_cps_putx16, 0, ra) + move s5, ra + move s4, a0 + srl a0, a0, 8 + jal _mips_cps_putx8 + move a0, s4 + move ra, s5 + b _mips_cps_putx8 + END(_mips_cps_putx16) + +/** + * _mips_cps_putx32 - write a 32b hex value to the UART + * @a0: the 32b value to write to the UART + * @t9: UART base address + * + * Write a 32 bit value (ie. 8 hexadecimal characters) to the UART. + */ +NESTED(_mips_cps_putx32, 0, ra) + move s7, ra + move s6, a0 + srl a0, a0, 16 + jal _mips_cps_putx16 + move a0, s6 + move ra, s7 + b _mips_cps_putx16 + END(_mips_cps_putx32) + +#ifdef CONFIG_64BIT + +/** + * _mips_cps_putx64 - write a 64b hex value to the UART + * @a0: the 64b value to write to the UART + * @t9: UART base address + * + * Write a 64 bit value (ie. 16 hexadecimal characters) to the UART. + */ +NESTED(_mips_cps_putx64, 0, ra) + move sp, ra + move s8, a0 + dsrl32 a0, a0, 0 + jal _mips_cps_putx32 + move a0, s8 + move ra, sp + b _mips_cps_putx32 + END(_mips_cps_putx64) + +#define _mips_cps_putxlong _mips_cps_putx64 + +#else /* !CONFIG_64BIT */ + +#define _mips_cps_putxlong _mips_cps_putx32 + +#endif /* !CONFIG_64BIT */ + +/** + * mips_cps_bev_dump() - dump relevant exception state to UART + * @a0: pointer to NULL-terminated ASCII string naming the exception + * + * Write information that may be useful in debugging an exception to the + * UART configured by CONFIG_MIPS_CPS_NS16550_*. As this BEV exception + * will only be run if something goes horribly wrong very early during + * the bringup of a core and it is very likely to be unsafe to perform + * memory accesses at that point (cache state indeterminate, EVA may not + * be configured, coherence may be disabled) let alone have a stack, + * this is all written in assembly using only registers & unmapped + * uncached access to the UART registers. + */ +LEAF(mips_cps_bev_dump) + move s0, ra + move s1, a0 + + li t9, CKSEG1ADDR(CONFIG_MIPS_CPS_NS16550_BASE) + + PTR_LA a0, str_newline + jal _mips_cps_puts + PTR_LA a0, str_bev + jal _mips_cps_puts + move a0, s1 + jal _mips_cps_puts + PTR_LA a0, str_newline + jal _mips_cps_puts + PTR_LA a0, str_newline + jal _mips_cps_puts + +#define DUMP_COP0_REG(reg, name, sz, _mfc0) \ + PTR_LA a0, 8f; \ + jal _mips_cps_puts; \ + _mfc0 a0, reg; \ + jal _mips_cps_putx##sz; \ + PTR_LA a0, str_newline; \ + jal _mips_cps_puts; \ + TEXT(name) + + DUMP_COP0_REG(CP0_CAUSE, "Cause: 0x", 32, mfc0) + DUMP_COP0_REG(CP0_STATUS, "Status: 0x", 32, mfc0) + DUMP_COP0_REG(CP0_EBASE, "EBase: 0x", long, MFC0) + DUMP_COP0_REG(CP0_BADVADDR, "BadVAddr: 0x", long, MFC0) + DUMP_COP0_REG(CP0_BADINSTR, "BadInstr: 0x", 32, mfc0) + + PTR_LA a0, str_newline + jal _mips_cps_puts + jr s0 + END(mips_cps_bev_dump) + +.pushsection .data +str_bev: .asciiz "BEV Exception: " +str_newline: .asciiz "\r\n" +.popsection diff --git a/arch/mips/kernel/cps-vec.S b/arch/mips/kernel/cps-vec.S new file mode 100644 index 000000000000..2ae7034a3d5c --- /dev/null +++ b/arch/mips/kernel/cps-vec.S @@ -0,0 +1,610 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2013 Imagination Technologies + * Author: Paul Burton <paul.burton@mips.com> + */ + +#include <linux/init.h> +#include <asm/addrspace.h> +#include <asm/asm.h> +#include <asm/asm-offsets.h> +#include <asm/asmmacro.h> +#include <asm/cacheops.h> +#include <asm/eva.h> +#include <asm/mipsregs.h> +#include <asm/mipsmtregs.h> +#include <asm/pm.h> +#include <asm/smp-cps.h> + +#define GCR_CPC_BASE_OFS 0x0088 +#define GCR_CL_COHERENCE_OFS 0x2008 +#define GCR_CL_ID_OFS 0x2028 +#define CM3_GCR_Cx_ID_CLUSTER_SHF 8 +#define CM3_GCR_Cx_ID_CLUSTER_MSK (0xff << 8) +#define CM3_GCR_Cx_ID_CORENUM_SHF 0 +#define CM3_GCR_Cx_ID_CORENUM_MSK (0xff << 0) + +#define CPC_CL_VC_STOP_OFS 0x2020 +#define CPC_CL_VC_RUN_OFS 0x2028 + +.extern mips_cm_base + +.set noreorder + +#ifdef CONFIG_64BIT +# define STATUS_BITDEPS ST0_KX +#else +# define STATUS_BITDEPS 0 +#endif + +#ifdef CONFIG_MIPS_CPS_NS16550 + +#define DUMP_EXCEP(name) \ + PTR_LA a0, 8f; \ + jal mips_cps_bev_dump; \ + nop; \ + TEXT(name) + +#else /* !CONFIG_MIPS_CPS_NS16550 */ + +#define DUMP_EXCEP(name) + +#endif /* !CONFIG_MIPS_CPS_NS16550 */ + + /* + * Set dest to non-zero if the core supports the MT ASE, else zero. If + * MT is not supported then branch to nomt. + */ + .macro has_mt dest, nomt + mfc0 \dest, CP0_CONFIG, 1 + bgez \dest, \nomt + mfc0 \dest, CP0_CONFIG, 2 + bgez \dest, \nomt + mfc0 \dest, CP0_CONFIG, 3 + andi \dest, \dest, MIPS_CONF3_MT + beqz \dest, \nomt + nop + .endm + + /* + * Set dest to non-zero if the core supports MIPSr6 multithreading + * (ie. VPs), else zero. If MIPSr6 multithreading is not supported then + * branch to nomt. + */ + .macro has_vp dest, nomt + mfc0 \dest, CP0_CONFIG, 1 + bgez \dest, \nomt + mfc0 \dest, CP0_CONFIG, 2 + bgez \dest, \nomt + mfc0 \dest, CP0_CONFIG, 3 + bgez \dest, \nomt + mfc0 \dest, CP0_CONFIG, 4 + bgez \dest, \nomt + mfc0 \dest, CP0_CONFIG, 5 + andi \dest, \dest, MIPS_CONF5_VP + beqz \dest, \nomt + nop + .endm + + +LEAF(mips_cps_core_boot) + /* Save CCA and GCR base */ + move s0, a0 + move s1, a1 + + /* We don't know how to do coherence setup on earlier ISA */ +#if MIPS_ISA_REV > 0 + /* Skip cache & coherence setup if we're already coherent */ + lw s7, GCR_CL_COHERENCE_OFS(s1) + bnez s7, 1f + nop + + /* Initialize the L1 caches */ + jal mips_cps_cache_init + nop + + /* Enter the coherent domain */ + li t0, 0xff + sw t0, GCR_CL_COHERENCE_OFS(s1) + ehb +#endif /* MIPS_ISA_REV > 0 */ + + /* Set Kseg0 CCA to that in s0 */ +1: mfc0 t0, CP0_CONFIG + ori t0, 0x7 + xori t0, 0x7 + or t0, t0, s0 + mtc0 t0, CP0_CONFIG + ehb + + /* Jump to kseg0 */ + PTR_LA t0, 1f + jr t0 + nop + + /* + * We're up, cached & coherent. Perform any EVA initialization necessary + * before we access memory. + */ +1: eva_init + + /* Retrieve boot configuration pointers */ + jal mips_cps_get_bootcfg + nop + + /* Skip core-level init if we started up coherent */ + bnez s7, 1f + nop + + /* Perform any further required core-level initialisation */ + jal mips_cps_core_init + nop + + /* + * Boot any other VPEs within this core that should be online, and + * deactivate this VPE if it should be offline. + */ + move a1, t9 + jal mips_cps_boot_vpes + move a0, v0 + + /* Off we go! */ +1: PTR_L t1, VPEBOOTCFG_PC(v1) + PTR_L gp, VPEBOOTCFG_GP(v1) + PTR_L sp, VPEBOOTCFG_SP(v1) + jr t1 + nop + END(mips_cps_core_boot) + + __INIT +LEAF(excep_tlbfill) + DUMP_EXCEP("TLB Fill") + b . + nop + END(excep_tlbfill) + +LEAF(excep_xtlbfill) + DUMP_EXCEP("XTLB Fill") + b . + nop + END(excep_xtlbfill) + +LEAF(excep_cache) + DUMP_EXCEP("Cache") + b . + nop + END(excep_cache) + +LEAF(excep_genex) + DUMP_EXCEP("General") + b . + nop + END(excep_genex) + +LEAF(excep_intex) + DUMP_EXCEP("Interrupt") + b . + nop + END(excep_intex) + +LEAF(excep_ejtag) + PTR_LA k0, ejtag_debug_handler + jr k0 + nop + END(excep_ejtag) + __FINIT + +LEAF(mips_cps_core_init) +#ifdef CONFIG_MIPS_MT_SMP + /* Check that the core implements the MT ASE */ + has_mt t0, 3f + + .set push + .set MIPS_ISA_LEVEL_RAW + .set mt + + /* Only allow 1 TC per VPE to execute... */ + dmt + + /* ...and for the moment only 1 VPE */ + dvpe + PTR_LA t1, 1f + jr.hb t1 + nop + + /* Enter VPE configuration state */ +1: mfc0 t0, CP0_MVPCONTROL + ori t0, t0, MVPCONTROL_VPC + mtc0 t0, CP0_MVPCONTROL + + /* Retrieve the number of VPEs within the core */ + mfc0 t0, CP0_MVPCONF0 + srl t0, t0, MVPCONF0_PVPE_SHIFT + andi t0, t0, (MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT) + addiu ta3, t0, 1 + + /* If there's only 1, we're done */ + beqz t0, 2f + nop + + /* Loop through each VPE within this core */ + li ta1, 1 + +1: /* Operate on the appropriate TC */ + mtc0 ta1, CP0_VPECONTROL + ehb + + /* Bind TC to VPE (1:1 TC:VPE mapping) */ + mttc0 ta1, CP0_TCBIND + + /* Set exclusive TC, non-active, master */ + li t0, VPECONF0_MVP + sll t1, ta1, VPECONF0_XTC_SHIFT + or t0, t0, t1 + mttc0 t0, CP0_VPECONF0 + + /* Set TC non-active, non-allocatable */ + mttc0 zero, CP0_TCSTATUS + + /* Set TC halted */ + li t0, TCHALT_H + mttc0 t0, CP0_TCHALT + + /* Next VPE */ + addiu ta1, ta1, 1 + slt t0, ta1, ta3 + bnez t0, 1b + nop + + /* Leave VPE configuration state */ +2: mfc0 t0, CP0_MVPCONTROL + xori t0, t0, MVPCONTROL_VPC + mtc0 t0, CP0_MVPCONTROL + +3: .set pop +#endif + jr ra + nop + END(mips_cps_core_init) + +/** + * mips_cps_get_bootcfg() - retrieve boot configuration pointers + * + * Returns: pointer to struct core_boot_config in v0, pointer to + * struct vpe_boot_config in v1, VPE ID in t9 + */ +LEAF(mips_cps_get_bootcfg) + /* Calculate a pointer to this cores struct core_boot_config */ + PTR_LA v0, mips_cps_cluster_bootcfg + PTR_L v0, 0(v0) + lw t0, GCR_CL_ID_OFS(s1) +#ifdef CONFIG_CPU_MIPSR6 + ext t1, t0, CM3_GCR_Cx_ID_CLUSTER_SHF, 8 + li t2, CLUSTERBOOTCFG_SIZE + mul t1, t1, t2 + PTR_ADDU \ + v0, v0, t1 +#endif + PTR_L v0, CLUSTERBOOTCFG_CORECONFIG(v0) + andi t0, t0, CM3_GCR_Cx_ID_CORENUM_MSK + li t1, COREBOOTCFG_SIZE + mul t0, t0, t1 + PTR_ADDU v0, v0, t0 + + /* Calculate this VPEs ID. If the core doesn't support MT use 0 */ + li t9, 0 +#if defined(CONFIG_CPU_MIPSR6) + has_vp ta2, 1f + + /* + * Assume non-contiguous numbering. Perhaps some day we'll need + * to handle contiguous VP numbering, but no such systems yet + * exist. + */ + mfc0 t9, CP0_GLOBALNUMBER + andi t9, t9, MIPS_GLOBALNUMBER_VP +#elif defined(CONFIG_MIPS_MT_SMP) + has_mt ta2, 1f + + /* Find the number of VPEs present in the core */ + mfc0 t1, CP0_MVPCONF0 + srl t1, t1, MVPCONF0_PVPE_SHIFT + andi t1, t1, MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT + addiu t1, t1, 1 + + /* Calculate a mask for the VPE ID from EBase.CPUNum */ + clz t1, t1 + li t2, 31 + subu t1, t2, t1 + li t2, 1 + sll t1, t2, t1 + addiu t1, t1, -1 + + /* Retrieve the VPE ID from EBase.CPUNum */ + mfc0 t9, $15, 1 + and t9, t9, t1 +#endif + +1: /* Calculate a pointer to this VPEs struct vpe_boot_config */ + li t1, VPEBOOTCFG_SIZE + mul v1, t9, t1 + PTR_L ta3, COREBOOTCFG_VPECONFIG(v0) + PTR_ADDU v1, v1, ta3 + + jr ra + nop + END(mips_cps_get_bootcfg) + +LEAF(mips_cps_boot_vpes) + lw ta2, COREBOOTCFG_VPEMASK(a0) + PTR_L ta3, COREBOOTCFG_VPECONFIG(a0) + +#if defined(CONFIG_CPU_MIPSR6) + + has_vp t0, 5f + + /* Find base address of CPC */ + PTR_LA t1, mips_gcr_base + PTR_L t1, 0(t1) + PTR_L t1, GCR_CPC_BASE_OFS(t1) + PTR_LI t2, ~0x7fff + and t1, t1, t2 + PTR_LI t2, UNCAC_BASE + PTR_ADD t1, t1, t2 + + /* Start any other VPs that ought to be running */ + PTR_S ta2, CPC_CL_VC_RUN_OFS(t1) + + /* Ensure this VP stops running if it shouldn't be */ + not ta2 + PTR_S ta2, CPC_CL_VC_STOP_OFS(t1) + ehb + +#elif defined(CONFIG_MIPS_MT) + + /* If the core doesn't support MT then return */ + has_mt t0, 5f + + /* Enter VPE configuration state */ + .set push + .set MIPS_ISA_LEVEL_RAW + .set mt + dvpe + .set pop + + PTR_LA t1, 1f + jr.hb t1 + nop +1: mfc0 t1, CP0_MVPCONTROL + ori t1, t1, MVPCONTROL_VPC + mtc0 t1, CP0_MVPCONTROL + ehb + + /* Loop through each VPE */ + move t8, ta2 + li ta1, 0 + + /* Check whether the VPE should be running. If not, skip it */ +1: andi t0, ta2, 1 + beqz t0, 2f + nop + + /* Operate on the appropriate TC */ + mfc0 t0, CP0_VPECONTROL + ori t0, t0, VPECONTROL_TARGTC + xori t0, t0, VPECONTROL_TARGTC + or t0, t0, ta1 + mtc0 t0, CP0_VPECONTROL + ehb + + .set push + .set MIPS_ISA_LEVEL_RAW + .set mt + + /* Skip the VPE if its TC is not halted */ + mftc0 t0, CP0_TCHALT + beqz t0, 2f + nop + + /* Calculate a pointer to the VPEs struct vpe_boot_config */ + li t0, VPEBOOTCFG_SIZE + mul t0, t0, ta1 + PTR_ADDU t0, t0, ta3 + + /* Set the TC restart PC */ + lw t1, VPEBOOTCFG_PC(t0) + mttc0 t1, CP0_TCRESTART + + /* Set the TC stack pointer */ + lw t1, VPEBOOTCFG_SP(t0) + mttgpr t1, sp + + /* Set the TC global pointer */ + lw t1, VPEBOOTCFG_GP(t0) + mttgpr t1, gp + + /* Copy config from this VPE */ + mfc0 t0, CP0_CONFIG + mttc0 t0, CP0_CONFIG + + /* + * Copy the EVA config from this VPE if the CPU supports it. + * CONFIG3 must exist to be running MT startup - just read it. + */ + mfc0 t0, CP0_CONFIG, 3 + and t0, t0, MIPS_CONF3_SC + beqz t0, 3f + nop + mfc0 t0, CP0_SEGCTL0 + mttc0 t0, CP0_SEGCTL0 + mfc0 t0, CP0_SEGCTL1 + mttc0 t0, CP0_SEGCTL1 + mfc0 t0, CP0_SEGCTL2 + mttc0 t0, CP0_SEGCTL2 +3: + /* Ensure no software interrupts are pending */ + mttc0 zero, CP0_CAUSE + mttc0 zero, CP0_STATUS + + /* Set TC active, not interrupt exempt */ + mftc0 t0, CP0_TCSTATUS + li t1, ~TCSTATUS_IXMT + and t0, t0, t1 + ori t0, t0, TCSTATUS_A + mttc0 t0, CP0_TCSTATUS + + /* Clear the TC halt bit */ + mttc0 zero, CP0_TCHALT + + /* Set VPE active */ + mftc0 t0, CP0_VPECONF0 + ori t0, t0, VPECONF0_VPA + mttc0 t0, CP0_VPECONF0 + + /* Next VPE */ +2: srl ta2, ta2, 1 + addiu ta1, ta1, 1 + bnez ta2, 1b + nop + + /* Leave VPE configuration state */ + mfc0 t1, CP0_MVPCONTROL + xori t1, t1, MVPCONTROL_VPC + mtc0 t1, CP0_MVPCONTROL + ehb + evpe + + .set pop + + /* Check whether this VPE is meant to be running */ + li t0, 1 + sll t0, t0, a1 + and t0, t0, t8 + bnez t0, 2f + nop + + /* This VPE should be offline, halt the TC */ + li t0, TCHALT_H + mtc0 t0, CP0_TCHALT + PTR_LA t0, 1f +1: jr.hb t0 + nop + +2: + +#endif /* CONFIG_MIPS_MT_SMP */ + + /* Return */ +5: jr ra + nop + END(mips_cps_boot_vpes) + +#if MIPS_ISA_REV > 0 +LEAF(mips_cps_cache_init) + /* + * Clear the bits used to index the caches. Note that the architecture + * dictates that writing to any of TagLo or TagHi selects 0 or 2 should + * be valid for all MIPS32 CPUs, even those for which said writes are + * unnecessary. + */ + mtc0 zero, CP0_TAGLO, 0 + mtc0 zero, CP0_TAGHI, 0 + mtc0 zero, CP0_TAGLO, 2 + mtc0 zero, CP0_TAGHI, 2 + ehb + + /* Primary cache configuration is indicated by Config1 */ + mfc0 v0, CP0_CONFIG, 1 + + /* Detect I-cache line size */ + _EXT t0, v0, MIPS_CONF1_IL_SHF, MIPS_CONF1_IL_SZ + beqz t0, icache_done + li t1, 2 + sllv t0, t1, t0 + + /* Detect I-cache size */ + _EXT t1, v0, MIPS_CONF1_IS_SHF, MIPS_CONF1_IS_SZ + xori t2, t1, 0x7 + beqz t2, 1f + li t3, 32 + addiu t1, t1, 1 + sllv t1, t3, t1 +1: /* At this point t1 == I-cache sets per way */ + _EXT t2, v0, MIPS_CONF1_IA_SHF, MIPS_CONF1_IA_SZ + addiu t2, t2, 1 + mul t1, t1, t0 + mul t1, t1, t2 + + li a0, CKSEG0 + PTR_ADD a1, a0, t1 +1: cache Index_Store_Tag_I, 0(a0) + PTR_ADD a0, a0, t0 + bne a0, a1, 1b + nop +icache_done: + + /* Detect D-cache line size */ + _EXT t0, v0, MIPS_CONF1_DL_SHF, MIPS_CONF1_DL_SZ + beqz t0, dcache_done + li t1, 2 + sllv t0, t1, t0 + + /* Detect D-cache size */ + _EXT t1, v0, MIPS_CONF1_DS_SHF, MIPS_CONF1_DS_SZ + xori t2, t1, 0x7 + beqz t2, 1f + li t3, 32 + addiu t1, t1, 1 + sllv t1, t3, t1 +1: /* At this point t1 == D-cache sets per way */ + _EXT t2, v0, MIPS_CONF1_DA_SHF, MIPS_CONF1_DA_SZ + addiu t2, t2, 1 + mul t1, t1, t0 + mul t1, t1, t2 + + li a0, CKSEG0 + PTR_ADDU a1, a0, t1 + PTR_SUBU a1, a1, t0 +1: cache Index_Store_Tag_D, 0(a0) + bne a0, a1, 1b + PTR_ADD a0, a0, t0 +dcache_done: + + jr ra + nop + END(mips_cps_cache_init) +#endif /* MIPS_ISA_REV > 0 */ + +#if defined(CONFIG_MIPS_CPS_PM) && defined(CONFIG_CPU_PM) + + /* Calculate a pointer to this CPUs struct mips_static_suspend_state */ + .macro psstate dest + .set push + .set noat + lw $1, TI_CPU(gp) + sll $1, $1, LONGLOG + PTR_LA \dest, __per_cpu_offset + PTR_ADDU $1, $1, \dest + lw $1, 0($1) + PTR_LA \dest, cps_cpu_state + PTR_ADDU \dest, \dest, $1 + .set pop + .endm + +LEAF(mips_cps_pm_save) + /* Save CPU state */ + SUSPEND_SAVE_REGS + psstate t1 + SUSPEND_SAVE_STATIC + jr v0 + nop + END(mips_cps_pm_save) + +LEAF(mips_cps_pm_restore) + /* Restore CPU state */ + psstate t1 + RESUME_RESTORE_STATIC + RESUME_RESTORE_REGS_RETURN + END(mips_cps_pm_restore) + +#endif /* CONFIG_MIPS_CPS_PM && CONFIG_CPU_PM */ diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index c7b1b3c5a761..1e49e05ac8b1 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Processor capabilities determination functions. * @@ -5,14 +6,10 @@ * Copyright (C) 1994 - 2006 Ralf Baechle * Copyright (C) 2003, 2004 Maciej W. Rozycki * Copyright (C) 2001, 2004, 2011, 2012 MIPS Technologies, Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #include <linux/init.h> #include <linux/kernel.h> +#include <linux/mmu_context.h> #include <linux/ptrace.h> #include <linux/smp.h> #include <linux/stddef.h> @@ -20,26 +17,43 @@ #include <asm/bugs.h> #include <asm/cpu.h> +#include <asm/cpu-features.h> +#include <asm/cpu-type.h> #include <asm/fpu.h> #include <asm/mipsregs.h> +#include <asm/mipsmtregs.h> +#include <asm/msa.h> #include <asm/watch.h> #include <asm/elf.h> +#include <asm/pgtable-bits.h> #include <asm/spram.h> -#include <asm/uaccess.h> +#include <asm/traps.h> +#include <linux/uaccess.h> -static int __cpuinitdata mips_fpu_disabled; +#include "fpu-probe.h" -static int __init fpu_disable(char *s) -{ - cpu_data[0].options &= ~MIPS_CPU_FPU; - mips_fpu_disabled = 1; +#include <asm/mach-loongson64/cpucfg-emul.h> - return 1; -} +/* Hardware capabilities */ +unsigned int elf_hwcap __read_mostly; +EXPORT_SYMBOL_GPL(elf_hwcap); -__setup("nofpu", fpu_disable); +static bool mmid_disabled_quirk; + +static inline unsigned long cpu_get_msa_id(void) +{ + unsigned long status, msa_id; + + status = read_c0_status(); + __enable_fpu(FPU_64BIT); + enable_msa(); + msa_id = read_msa_ir(); + disable_msa(); + write_c0_status(status); + return msa_id; +} -int __cpuinitdata mips_dsp_disabled; +static int mips_dsp_disabled; static int __init dsp_disable(char *s) { @@ -51,16 +65,101 @@ static int __init dsp_disable(char *s) __setup("nodsp", dsp_disable); +static int mips_htw_disabled; + +static int __init htw_disable(char *s) +{ + mips_htw_disabled = 1; + cpu_data[0].options &= ~MIPS_CPU_HTW; + write_c0_pwctl(read_c0_pwctl() & + ~(1 << MIPS_PWCTL_PWEN_SHIFT)); + + return 1; +} + +__setup("nohtw", htw_disable); + +static int mips_ftlb_disabled; +static int mips_has_ftlb_configured; + +enum ftlb_flags { + FTLB_EN = 1 << 0, + FTLB_SET_PROB = 1 << 1, +}; + +static int set_ftlb_enable(struct cpuinfo_mips *c, enum ftlb_flags flags); + +static int __init ftlb_disable(char *s) +{ + unsigned int config4, mmuextdef; + + /* + * If the core hasn't done any FTLB configuration, there is nothing + * for us to do here. + */ + if (!mips_has_ftlb_configured) + return 1; + + /* Disable it in the boot cpu */ + if (set_ftlb_enable(&cpu_data[0], 0)) { + pr_warn("Can't turn FTLB off\n"); + return 1; + } + + config4 = read_c0_config4(); + + /* Check that FTLB has been disabled */ + mmuextdef = config4 & MIPS_CONF4_MMUEXTDEF; + /* MMUSIZEEXT == VTLB ON, FTLB OFF */ + if (mmuextdef == MIPS_CONF4_MMUEXTDEF_FTLBSIZEEXT) { + /* This should never happen */ + pr_warn("FTLB could not be disabled!\n"); + return 1; + } + + mips_ftlb_disabled = 1; + mips_has_ftlb_configured = 0; + + /* + * noftlb is mainly used for debug purposes so print + * an informative message instead of using pr_debug() + */ + pr_info("FTLB has been disabled\n"); + + /* + * Some of these bits are duplicated in the decode_config4. + * MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT is the only possible case + * once FTLB has been disabled so undo what decode_config4 did. + */ + cpu_data[0].tlbsize -= cpu_data[0].tlbsizeftlbways * + cpu_data[0].tlbsizeftlbsets; + cpu_data[0].tlbsizeftlbsets = 0; + cpu_data[0].tlbsizeftlbways = 0; + + return 1; +} + +__setup("noftlb", ftlb_disable); + +/* + * Check if the CPU has per tc perf counters + */ +static inline void cpu_set_mt_per_tc_perf(struct cpuinfo_mips *c) +{ + if (read_c0_config7() & MTI_CONF7_PTC) + c->options |= MIPS_CPU_MT_PER_TC_PERF_COUNTERS; +} + static inline void check_errata(void) { struct cpuinfo_mips *c = ¤t_cpu_data; - switch (c->cputype) { + switch (current_cpu_type()) { case CPU_34K: /* * Erratum "RPS May Cause Incorrect Instruction Execution" - * This code only handles VPE0, any SMP/SMTC/RTOS code - * making use of VPE1 will be responsable for that VPE. + * This code only handles VPE0, any SMP/RTOS code + * making use of VPE1 will be responsible for that VPE. */ if ((c->processor_id & PRID_REV_MASK) <= PRID_REV_34K_V1_0_2) write_c0_config7(read_c0_config7() | MIPS_CONF7_RPS); @@ -83,7 +182,6 @@ void __init check_bugs32(void) static inline int cpu_has_confreg(void) { #ifdef CONFIG_CPU_R3000 - extern unsigned long r3k_cache_size(unsigned long); unsigned long size1, size2; unsigned long cfg = read_c0_conf(); @@ -103,26 +201,11 @@ static inline void set_elf_platform(int cpu, const char *plat) __elf_platform = plat; } -/* - * Get the FPU Implementation/Revision. - */ -static inline unsigned long cpu_get_fpu_id(void) -{ - unsigned long tmp, fpu_id; - - tmp = read_c0_status(); - __enable_fpu(); - fpu_id = read_32bit_cp1_register(CP1_REVISION); - write_c0_status(tmp); - return fpu_id; -} - -/* - * Check the CPU has an FPU the official way. - */ -static inline int __cpu_has_fpu(void) +static inline void set_elf_base_platform(const char *plat) { - return ((cpu_get_fpu_id() & 0xff00) != FPIR_IMP_NONE); + if (__elf_base_platform == NULL) { + __elf_base_platform = plat; + } } static inline void cpu_probe_vmbits(struct cpuinfo_mips *c) @@ -134,43 +217,206 @@ static inline void cpu_probe_vmbits(struct cpuinfo_mips *c) #endif } -static void __cpuinit set_isa(struct cpuinfo_mips *c, unsigned int isa) +static void set_isa(struct cpuinfo_mips *c, unsigned int isa) { switch (isa) { + case MIPS_CPU_ISA_M64R5: + c->isa_level |= MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5; + set_elf_base_platform("mips64r5"); + fallthrough; case MIPS_CPU_ISA_M64R2: c->isa_level |= MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2; + set_elf_base_platform("mips64r2"); + fallthrough; case MIPS_CPU_ISA_M64R1: c->isa_level |= MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1; + set_elf_base_platform("mips64"); + fallthrough; case MIPS_CPU_ISA_V: c->isa_level |= MIPS_CPU_ISA_V; + set_elf_base_platform("mips5"); + fallthrough; case MIPS_CPU_ISA_IV: c->isa_level |= MIPS_CPU_ISA_IV; + set_elf_base_platform("mips4"); + fallthrough; case MIPS_CPU_ISA_III: c->isa_level |= MIPS_CPU_ISA_II | MIPS_CPU_ISA_III; - break; - + set_elf_base_platform("mips3"); + break; + + /* R6 incompatible with everything else */ + case MIPS_CPU_ISA_M64R6: + c->isa_level |= MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6; + set_elf_base_platform("mips64r6"); + fallthrough; + case MIPS_CPU_ISA_M32R6: + c->isa_level |= MIPS_CPU_ISA_M32R6; + set_elf_base_platform("mips32r6"); + /* Break here so we don't add incompatible ISAs */ + break; + case MIPS_CPU_ISA_M32R5: + c->isa_level |= MIPS_CPU_ISA_M32R5; + set_elf_base_platform("mips32r5"); + fallthrough; case MIPS_CPU_ISA_M32R2: c->isa_level |= MIPS_CPU_ISA_M32R2; + set_elf_base_platform("mips32r2"); + fallthrough; case MIPS_CPU_ISA_M32R1: c->isa_level |= MIPS_CPU_ISA_M32R1; + set_elf_base_platform("mips32"); + fallthrough; case MIPS_CPU_ISA_II: c->isa_level |= MIPS_CPU_ISA_II; + set_elf_base_platform("mips2"); break; } } -static char unknown_isa[] __cpuinitdata = KERN_ERR \ +static char unknown_isa[] = KERN_ERR \ "Unsupported ISA type, c0.config0: %d."; +static unsigned int calculate_ftlb_probability(struct cpuinfo_mips *c) +{ + + unsigned int probability = c->tlbsize / c->tlbsizevtlb; + + /* + * 0 = All TLBWR instructions go to FTLB + * 1 = 15:1: For every 16 TBLWR instructions, 15 go to the + * FTLB and 1 goes to the VTLB. + * 2 = 7:1: As above with 7:1 ratio. + * 3 = 3:1: As above with 3:1 ratio. + * + * Use the linear midpoint as the probability threshold. + */ + if (probability >= 12) + return 1; + else if (probability >= 6) + return 2; + else + /* + * So FTLB is less than 4 times bigger than VTLB. + * A 3:1 ratio can still be useful though. + */ + return 3; +} + +static int set_ftlb_enable(struct cpuinfo_mips *c, enum ftlb_flags flags) +{ + unsigned int config; + + /* It's implementation dependent how the FTLB can be enabled */ + switch (c->cputype) { + case CPU_PROAPTIV: + case CPU_P5600: + case CPU_P6600: + /* proAptiv & related cores use Config6 to enable the FTLB */ + config = read_c0_config6(); + + if (flags & FTLB_EN) + config |= MTI_CONF6_FTLBEN; + else + config &= ~MTI_CONF6_FTLBEN; + + if (flags & FTLB_SET_PROB) { + config &= ~(3 << MTI_CONF6_FTLBP_SHIFT); + config |= calculate_ftlb_probability(c) + << MTI_CONF6_FTLBP_SHIFT; + } + + write_c0_config6(config); + back_to_back_c0_hazard(); + break; + case CPU_I6400: + case CPU_I6500: + /* There's no way to disable the FTLB */ + if (!(flags & FTLB_EN)) + return 1; + return 0; + case CPU_LOONGSON64: + /* Flush ITLB, DTLB, VTLB and FTLB */ + write_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB | + LOONGSON_DIAG_VTLB | LOONGSON_DIAG_FTLB); + /* Loongson-3 cores use Config6 to enable the FTLB */ + config = read_c0_config6(); + if (flags & FTLB_EN) + /* Enable FTLB */ + write_c0_config6(config & ~LOONGSON_CONF6_FTLBDIS); + else + /* Disable FTLB */ + write_c0_config6(config | LOONGSON_CONF6_FTLBDIS); + break; + default: + return 1; + } + + return 0; +} + +static int mm_config(struct cpuinfo_mips *c) +{ + unsigned int config0, update, mm; + + config0 = read_c0_config(); + mm = config0 & MIPS_CONF_MM; + + /* + * It's implementation dependent what type of write-merge is supported + * and whether it can be enabled/disabled. If it is settable lets make + * the merging allowed by default. Some platforms might have + * write-through caching unsupported. In this case just ignore the + * CP0.Config.MM bit field value. + */ + switch (c->cputype) { + case CPU_24K: + case CPU_34K: + case CPU_74K: + case CPU_P5600: + case CPU_P6600: + c->options |= MIPS_CPU_MM_FULL; + update = MIPS_CONF_MM_FULL; + break; + case CPU_1004K: + case CPU_1074K: + case CPU_INTERAPTIV: + case CPU_PROAPTIV: + mm = 0; + fallthrough; + default: + update = 0; + break; + } + + if (update) { + config0 = (config0 & ~MIPS_CONF_MM) | update; + write_c0_config(config0); + } else if (mm == MIPS_CONF_MM_SYSAD) { + c->options |= MIPS_CPU_MM_SYSAD; + } else if (mm == MIPS_CONF_MM_FULL) { + c->options |= MIPS_CPU_MM_FULL; + } + + return 0; +} + static inline unsigned int decode_config0(struct cpuinfo_mips *c) { unsigned int config0; - int isa; + int isa, mt; config0 = read_c0_config(); - if (((config0 & MIPS_CONF_MT) >> 7) == 1) + /* + * Look for Standard TLB or Dual VTLB and FTLB + */ + mt = config0 & MIPS_CONF_MT; + if (mt == MIPS_CONF_MT_TLB) c->options |= MIPS_CPU_TLB; + else if (mt == MIPS_CONF_MT_FTLB) + c->options |= MIPS_CPU_TLB | MIPS_CPU_FTLB; + isa = (config0 & MIPS_CONF_AT) >> 13; switch (isa) { case 0: @@ -181,6 +427,9 @@ static inline unsigned int decode_config0(struct cpuinfo_mips *c) case 1: set_isa(c, MIPS_CPU_ISA_M32R2); break; + case 2: + set_isa(c, MIPS_CPU_ISA_M32R6); + break; default: goto unknown; } @@ -193,6 +442,9 @@ static inline unsigned int decode_config0(struct cpuinfo_mips *c) case 1: set_isa(c, MIPS_CPU_ISA_M64R2); break; + case 2: + set_isa(c, MIPS_CPU_ISA_M64R6); + break; default: goto unknown; } @@ -215,6 +467,8 @@ static inline unsigned int decode_config1(struct cpuinfo_mips *c) if (config1 & MIPS_CONF1_MD) c->ases |= MIPS_ASE_MDMX; + if (config1 & MIPS_CONF1_PC) + c->options |= MIPS_CPU_PERF; if (config1 & MIPS_CONF1_WR) c->options |= MIPS_CPU_WATCH; if (config1 & MIPS_CONF1_CA) @@ -225,8 +479,11 @@ static inline unsigned int decode_config1(struct cpuinfo_mips *c) c->options |= MIPS_CPU_FPU; c->options |= MIPS_CPU_32FPR; } - if (cpu_has_tlb) + if (cpu_has_tlb) { c->tlbsize = ((config1 & MIPS_CONF1_TLBS) >> 25) + 1; + c->tlbsizevtlb = c->tlbsize; + c->tlbsizeftlbsets = 0; + } return config1 & MIPS_CONF_M; } @@ -251,18 +508,25 @@ static inline unsigned int decode_config3(struct cpuinfo_mips *c) if (config3 & MIPS_CONF3_SM) { c->ases |= MIPS_ASE_SMARTMIPS; - c->options |= MIPS_CPU_RIXI; + c->options |= MIPS_CPU_RIXI | MIPS_CPU_CTXTC; } if (config3 & MIPS_CONF3_RXI) c->options |= MIPS_CPU_RIXI; + if (config3 & MIPS_CONF3_CTXTC) + c->options |= MIPS_CPU_CTXTC; if (config3 & MIPS_CONF3_DSP) c->ases |= MIPS_ASE_DSP; - if (config3 & MIPS_CONF3_DSP2P) + if (config3 & MIPS_CONF3_DSP2P) { c->ases |= MIPS_ASE_DSP2P; + if (cpu_has_mips_r6) + c->ases |= MIPS_ASE_DSP3; + } if (config3 & MIPS_CONF3_VINT) c->options |= MIPS_CPU_VINT; if (config3 & MIPS_CONF3_VEIC) c->options |= MIPS_CPU_VEIC; + if (config3 & MIPS_CONF3_LPA) + c->options |= MIPS_CPU_LPA; if (config3 & MIPS_CONF3_MT) c->ases |= MIPS_ASE_MIPSMT; if (config3 & MIPS_CONF3_ULRI) @@ -271,6 +535,22 @@ static inline unsigned int decode_config3(struct cpuinfo_mips *c) c->options |= MIPS_CPU_MICROMIPS; if (config3 & MIPS_CONF3_VZ) c->ases |= MIPS_ASE_VZ; + if (config3 & MIPS_CONF3_SC) + c->options |= MIPS_CPU_SEGMENTS; + if (config3 & MIPS_CONF3_BI) + c->options |= MIPS_CPU_BADINSTR; + if (config3 & MIPS_CONF3_BP) + c->options |= MIPS_CPU_BADINSTRP; + if (config3 & MIPS_CONF3_MSA) + c->ases |= MIPS_ASE_MSA; + if (config3 & MIPS_CONF3_PW) { + c->htw_seq = 0; + c->options |= MIPS_CPU_HTW; + } + if (config3 & MIPS_CONF3_CDMM) + c->options |= MIPS_CPU_CDMM; + if (config3 & MIPS_CONF3_SP) + c->options |= MIPS_CPU_SP; return config3 & MIPS_CONF_M; } @@ -278,19 +558,167 @@ static inline unsigned int decode_config3(struct cpuinfo_mips *c) static inline unsigned int decode_config4(struct cpuinfo_mips *c) { unsigned int config4; + unsigned int newcf4; + unsigned int mmuextdef; + unsigned int ftlb_page = MIPS_CONF4_FTLBPAGESIZE; + unsigned long asid_mask; config4 = read_c0_config4(); - if ((config4 & MIPS_CONF4_MMUEXTDEF) == MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT - && cpu_has_tlb) - c->tlbsize += (config4 & MIPS_CONF4_MMUSIZEEXT) * 0x40; + if (cpu_has_tlb) { + if (((config4 & MIPS_CONF4_IE) >> 29) == 2) + c->options |= MIPS_CPU_TLBINV; - c->kscratch_mask = (config4 >> 16) & 0xff; + /* + * R6 has dropped the MMUExtDef field from config4. + * On R6 the fields always describe the FTLB, and only if it is + * present according to Config.MT. + */ + if (!cpu_has_mips_r6) + mmuextdef = config4 & MIPS_CONF4_MMUEXTDEF; + else if (cpu_has_ftlb) + mmuextdef = MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT; + else + mmuextdef = 0; + + switch (mmuextdef) { + case MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT: + c->tlbsize += (config4 & MIPS_CONF4_MMUSIZEEXT) * 0x40; + c->tlbsizevtlb = c->tlbsize; + break; + case MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT: + c->tlbsizevtlb += + ((config4 & MIPS_CONF4_VTLBSIZEEXT) >> + MIPS_CONF4_VTLBSIZEEXT_SHIFT) * 0x40; + c->tlbsize = c->tlbsizevtlb; + ftlb_page = MIPS_CONF4_VFTLBPAGESIZE; + fallthrough; + case MIPS_CONF4_MMUEXTDEF_FTLBSIZEEXT: + if (mips_ftlb_disabled) + break; + newcf4 = (config4 & ~ftlb_page) | + (page_size_ftlb(mmuextdef) << + MIPS_CONF4_FTLBPAGESIZE_SHIFT); + write_c0_config4(newcf4); + back_to_back_c0_hazard(); + config4 = read_c0_config4(); + if (config4 != newcf4) { + pr_err("PAGE_SIZE 0x%lx is not supported by FTLB (config4=0x%x)\n", + PAGE_SIZE, config4); + /* Switch FTLB off */ + set_ftlb_enable(c, 0); + mips_ftlb_disabled = 1; + break; + } + c->tlbsizeftlbsets = 1 << + ((config4 & MIPS_CONF4_FTLBSETS) >> + MIPS_CONF4_FTLBSETS_SHIFT); + c->tlbsizeftlbways = ((config4 & MIPS_CONF4_FTLBWAYS) >> + MIPS_CONF4_FTLBWAYS_SHIFT) + 2; + c->tlbsize += c->tlbsizeftlbways * c->tlbsizeftlbsets; + mips_has_ftlb_configured = 1; + break; + } + } + + c->kscratch_mask = (config4 & MIPS_CONF4_KSCREXIST) + >> MIPS_CONF4_KSCREXIST_SHIFT; + + asid_mask = MIPS_ENTRYHI_ASID; + if (config4 & MIPS_CONF4_AE) + asid_mask |= MIPS_ENTRYHI_ASIDX; + set_cpu_asid_mask(c, asid_mask); + + /* + * Warn if the computed ASID mask doesn't match the mask the kernel + * is built for. This may indicate either a serious problem or an + * easy optimisation opportunity, but either way should be addressed. + */ + WARN_ON(asid_mask != cpu_asid_mask(c)); return config4 & MIPS_CONF_M; } -static void __cpuinit decode_configs(struct cpuinfo_mips *c) +static inline unsigned int decode_config5(struct cpuinfo_mips *c) +{ + unsigned int config5, max_mmid_width; + unsigned long asid_mask; + + config5 = read_c0_config5(); + config5 &= ~(MIPS_CONF5_UFR | MIPS_CONF5_UFE); + + if (cpu_has_mips_r6) { + if (!mmid_disabled_quirk && (!__builtin_constant_p(cpu_has_mmid) || cpu_has_mmid)) + config5 |= MIPS_CONF5_MI; + else + config5 &= ~MIPS_CONF5_MI; + } + + write_c0_config5(config5); + + if (config5 & MIPS_CONF5_EVA) + c->options |= MIPS_CPU_EVA; + if (config5 & MIPS_CONF5_MRP) + c->options |= MIPS_CPU_MAAR; + if (config5 & MIPS_CONF5_LLB) + c->options |= MIPS_CPU_RW_LLB; + if (config5 & MIPS_CONF5_MVH) + c->options |= MIPS_CPU_MVH; + if (cpu_has_mips_r6 && (config5 & MIPS_CONF5_VP)) + c->options |= MIPS_CPU_VP; + if (config5 & MIPS_CONF5_CA2) + c->ases |= MIPS_ASE_MIPS16E2; + + if (config5 & MIPS_CONF5_CRCP) + elf_hwcap |= HWCAP_MIPS_CRC32; + + if (cpu_has_mips_r6) { + /* Ensure the write to config5 above takes effect */ + back_to_back_c0_hazard(); + + /* Check whether we successfully enabled MMID support */ + config5 = read_c0_config5(); + if (config5 & MIPS_CONF5_MI) + c->options |= MIPS_CPU_MMID; + + /* + * Warn if we've hardcoded cpu_has_mmid to a value unsuitable + * for the CPU we're running on, or if CPUs in an SMP system + * have inconsistent MMID support. + */ + WARN_ON(!!cpu_has_mmid != !!(config5 & MIPS_CONF5_MI)); + + if (cpu_has_mmid) { + write_c0_memorymapid(~0ul); + back_to_back_c0_hazard(); + asid_mask = read_c0_memorymapid(); + + /* + * We maintain a bitmap to track MMID allocation, and + * need a sensible upper bound on the size of that + * bitmap. The initial CPU with MMID support (I6500) + * supports 16 bit MMIDs, which gives us an 8KiB + * bitmap. The architecture recommends that hardware + * support 32 bit MMIDs, which would give us a 512MiB + * bitmap - that's too big in most cases. + * + * Cap MMID width at 16 bits for now & we can revisit + * this if & when hardware supports anything wider. + */ + max_mmid_width = 16; + if (asid_mask > GENMASK(max_mmid_width - 1, 0)) { + pr_info("Capping MMID width at %d bits", + max_mmid_width); + asid_mask = GENMASK(max_mmid_width - 1, 0); + } + set_cpu_asid_mask(c, asid_mask); + } + } + + return config5 & MIPS_CONF_M; +} + +static void decode_configs(struct cpuinfo_mips *c) { int ok; @@ -300,6 +728,9 @@ static void __cpuinit decode_configs(struct cpuinfo_mips *c) c->scache.flags = MIPS_CACHE_NOT_PRESENT; + /* Enable FTLB if present and not disabled */ + set_ftlb_enable(c, mips_ftlb_disabled ? 0 : FTLB_EN); + ok = decode_config0(c); /* Read Config registers. */ BUG_ON(!ok); /* Arch spec violation! */ if (ok) @@ -310,11 +741,295 @@ static void __cpuinit decode_configs(struct cpuinfo_mips *c) ok = decode_config3(c); if (ok) ok = decode_config4(c); + if (ok) + ok = decode_config5(c); + + /* Probe the EBase.WG bit */ + if (cpu_has_mips_r2_r6) { + u64 ebase; + unsigned int status; + + /* {read,write}_c0_ebase_64() may be UNDEFINED prior to r6 */ + ebase = cpu_has_mips64r6 ? read_c0_ebase_64() + : (s32)read_c0_ebase(); + if (ebase & MIPS_EBASE_WG) { + /* WG bit already set, we can avoid the clumsy probe */ + c->options |= MIPS_CPU_EBASE_WG; + } else { + /* Its UNDEFINED to change EBase while BEV=0 */ + status = read_c0_status(); + write_c0_status(status | ST0_BEV); + irq_enable_hazard(); + /* + * On pre-r6 cores, this may well clobber the upper bits + * of EBase. This is hard to avoid without potentially + * hitting UNDEFINED dm*c0 behaviour if EBase is 32-bit. + */ + if (cpu_has_mips64r6) + write_c0_ebase_64(ebase | MIPS_EBASE_WG); + else + write_c0_ebase(ebase | MIPS_EBASE_WG); + back_to_back_c0_hazard(); + /* Restore BEV */ + write_c0_status(status); + if (read_c0_ebase() & MIPS_EBASE_WG) { + c->options |= MIPS_CPU_EBASE_WG; + write_c0_ebase(ebase); + } + } + } + + /* configure the FTLB write probability */ + set_ftlb_enable(c, (mips_ftlb_disabled ? 0 : FTLB_EN) | FTLB_SET_PROB); mips_probe_watch_registers(c); - if (cpu_has_mips_r2) - c->core = read_c0_ebase() & 0x3ff; +#ifndef CONFIG_MIPS_CPS + if (cpu_has_mips_r2_r6) { + unsigned int core; + + core = get_ebase_cpunum(); + if (cpu_has_mipsmt) + core >>= fls(core_nvpes()) - 1; + cpu_set_core(c, core); + } +#endif +} + +/* + * Probe for certain guest capabilities by writing config bits and reading back. + * Finally write back the original value. + */ +#define probe_gc0_config(name, maxconf, bits) \ +do { \ + unsigned int tmp; \ + tmp = read_gc0_##name(); \ + write_gc0_##name(tmp | (bits)); \ + back_to_back_c0_hazard(); \ + maxconf = read_gc0_##name(); \ + write_gc0_##name(tmp); \ +} while (0) + +/* + * Probe for dynamic guest capabilities by changing certain config bits and + * reading back to see if they change. Finally write back the original value. + */ +#define probe_gc0_config_dyn(name, maxconf, dynconf, bits) \ +do { \ + maxconf = read_gc0_##name(); \ + write_gc0_##name(maxconf ^ (bits)); \ + back_to_back_c0_hazard(); \ + dynconf = maxconf ^ read_gc0_##name(); \ + write_gc0_##name(maxconf); \ + maxconf |= dynconf; \ +} while (0) + +static inline unsigned int decode_guest_config0(struct cpuinfo_mips *c) +{ + unsigned int config0; + + probe_gc0_config(config, config0, MIPS_CONF_M); + + if (config0 & MIPS_CONF_M) + c->guest.conf |= BIT(1); + return config0 & MIPS_CONF_M; +} + +static inline unsigned int decode_guest_config1(struct cpuinfo_mips *c) +{ + unsigned int config1, config1_dyn; + + probe_gc0_config_dyn(config1, config1, config1_dyn, + MIPS_CONF_M | MIPS_CONF1_PC | MIPS_CONF1_WR | + MIPS_CONF1_FP); + + if (config1 & MIPS_CONF1_FP) + c->guest.options |= MIPS_CPU_FPU; + if (config1_dyn & MIPS_CONF1_FP) + c->guest.options_dyn |= MIPS_CPU_FPU; + + if (config1 & MIPS_CONF1_WR) + c->guest.options |= MIPS_CPU_WATCH; + if (config1_dyn & MIPS_CONF1_WR) + c->guest.options_dyn |= MIPS_CPU_WATCH; + + if (config1 & MIPS_CONF1_PC) + c->guest.options |= MIPS_CPU_PERF; + if (config1_dyn & MIPS_CONF1_PC) + c->guest.options_dyn |= MIPS_CPU_PERF; + + if (config1 & MIPS_CONF_M) + c->guest.conf |= BIT(2); + return config1 & MIPS_CONF_M; +} + +static inline unsigned int decode_guest_config2(struct cpuinfo_mips *c) +{ + unsigned int config2; + + probe_gc0_config(config2, config2, MIPS_CONF_M); + + if (config2 & MIPS_CONF_M) + c->guest.conf |= BIT(3); + return config2 & MIPS_CONF_M; +} + +static inline unsigned int decode_guest_config3(struct cpuinfo_mips *c) +{ + unsigned int config3, config3_dyn; + + probe_gc0_config_dyn(config3, config3, config3_dyn, + MIPS_CONF_M | MIPS_CONF3_MSA | MIPS_CONF3_ULRI | + MIPS_CONF3_CTXTC); + + if (config3 & MIPS_CONF3_CTXTC) + c->guest.options |= MIPS_CPU_CTXTC; + if (config3_dyn & MIPS_CONF3_CTXTC) + c->guest.options_dyn |= MIPS_CPU_CTXTC; + + if (config3 & MIPS_CONF3_PW) + c->guest.options |= MIPS_CPU_HTW; + + if (config3 & MIPS_CONF3_ULRI) + c->guest.options |= MIPS_CPU_ULRI; + + if (config3 & MIPS_CONF3_SC) + c->guest.options |= MIPS_CPU_SEGMENTS; + + if (config3 & MIPS_CONF3_BI) + c->guest.options |= MIPS_CPU_BADINSTR; + if (config3 & MIPS_CONF3_BP) + c->guest.options |= MIPS_CPU_BADINSTRP; + + if (config3 & MIPS_CONF3_MSA) + c->guest.ases |= MIPS_ASE_MSA; + if (config3_dyn & MIPS_CONF3_MSA) + c->guest.ases_dyn |= MIPS_ASE_MSA; + + if (config3 & MIPS_CONF_M) + c->guest.conf |= BIT(4); + return config3 & MIPS_CONF_M; +} + +static inline unsigned int decode_guest_config4(struct cpuinfo_mips *c) +{ + unsigned int config4; + + probe_gc0_config(config4, config4, + MIPS_CONF_M | MIPS_CONF4_KSCREXIST); + + c->guest.kscratch_mask = (config4 & MIPS_CONF4_KSCREXIST) + >> MIPS_CONF4_KSCREXIST_SHIFT; + + if (config4 & MIPS_CONF_M) + c->guest.conf |= BIT(5); + return config4 & MIPS_CONF_M; +} + +static inline unsigned int decode_guest_config5(struct cpuinfo_mips *c) +{ + unsigned int config5, config5_dyn; + + probe_gc0_config_dyn(config5, config5, config5_dyn, + MIPS_CONF_M | MIPS_CONF5_MVH | MIPS_CONF5_MRP); + + if (config5 & MIPS_CONF5_MRP) + c->guest.options |= MIPS_CPU_MAAR; + if (config5_dyn & MIPS_CONF5_MRP) + c->guest.options_dyn |= MIPS_CPU_MAAR; + + if (config5 & MIPS_CONF5_LLB) + c->guest.options |= MIPS_CPU_RW_LLB; + + if (config5 & MIPS_CONF5_MVH) + c->guest.options |= MIPS_CPU_MVH; + + if (config5 & MIPS_CONF_M) + c->guest.conf |= BIT(6); + return config5 & MIPS_CONF_M; +} + +static inline void decode_guest_configs(struct cpuinfo_mips *c) +{ + unsigned int ok; + + ok = decode_guest_config0(c); + if (ok) + ok = decode_guest_config1(c); + if (ok) + ok = decode_guest_config2(c); + if (ok) + ok = decode_guest_config3(c); + if (ok) + ok = decode_guest_config4(c); + if (ok) + decode_guest_config5(c); +} + +static inline void cpu_probe_guestctl0(struct cpuinfo_mips *c) +{ + unsigned int guestctl0, temp; + + guestctl0 = read_c0_guestctl0(); + + if (guestctl0 & MIPS_GCTL0_G0E) + c->options |= MIPS_CPU_GUESTCTL0EXT; + if (guestctl0 & MIPS_GCTL0_G1) + c->options |= MIPS_CPU_GUESTCTL1; + if (guestctl0 & MIPS_GCTL0_G2) + c->options |= MIPS_CPU_GUESTCTL2; + if (!(guestctl0 & MIPS_GCTL0_RAD)) { + c->options |= MIPS_CPU_GUESTID; + + /* + * Probe for Direct Root to Guest (DRG). Set GuestCtl1.RID = 0 + * first, otherwise all data accesses will be fully virtualised + * as if they were performed by guest mode. + */ + write_c0_guestctl1(0); + tlbw_use_hazard(); + + write_c0_guestctl0(guestctl0 | MIPS_GCTL0_DRG); + back_to_back_c0_hazard(); + temp = read_c0_guestctl0(); + + if (temp & MIPS_GCTL0_DRG) { + write_c0_guestctl0(guestctl0); + c->options |= MIPS_CPU_DRG; + } + } +} + +static inline void cpu_probe_guestctl1(struct cpuinfo_mips *c) +{ + if (cpu_has_guestid) { + /* determine the number of bits of GuestID available */ + write_c0_guestctl1(MIPS_GCTL1_ID); + back_to_back_c0_hazard(); + c->guestid_mask = (read_c0_guestctl1() & MIPS_GCTL1_ID) + >> MIPS_GCTL1_ID_SHIFT; + write_c0_guestctl1(0); + } +} + +static inline void cpu_probe_gtoffset(struct cpuinfo_mips *c) +{ + /* determine the number of bits of GTOffset available */ + write_c0_gtoffset(0xffffffff); + back_to_back_c0_hazard(); + c->gtoffset_mask = read_c0_gtoffset(); + write_c0_gtoffset(0); +} + +static inline void cpu_probe_vz(struct cpuinfo_mips *c) +{ + cpu_probe_guestctl0(c); + if (cpu_has_guestctl1) + cpu_probe_guestctl1(c); + + cpu_probe_gtoffset(c); + + decode_guest_configs(c); } #define R4K_OPTS (MIPS_CPU_TLB | MIPS_CPU_4KEX | MIPS_CPU_4K_CACHE \ @@ -322,10 +1037,11 @@ static void __cpuinit decode_configs(struct cpuinfo_mips *c) static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) { - switch (c->processor_id & 0xff00) { + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_R2000: c->cputype = CPU_R2000; __cpu_name[cpu] = "R2000"; + c->fpu_msk31 |= FPU_CSR_CONDX | FPU_CSR_FS; c->options = MIPS_CPU_TLB | MIPS_CPU_3K_CACHE | MIPS_CPU_NOFPUEX; if (__cpu_has_fpu()) @@ -333,7 +1049,7 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) c->tlbsize = 64; break; case PRID_IMP_R3000: - if ((c->processor_id & 0xff) == PRID_REV_R3000A) { + if ((c->processor_id & PRID_REV_MASK) == PRID_REV_R3000A) { if (cpu_has_confreg()) { c->cputype = CPU_R3081E; __cpu_name[cpu] = "R3081"; @@ -345,6 +1061,7 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) c->cputype = CPU_R3000; __cpu_name[cpu] = "R3000"; } + c->fpu_msk31 |= FPU_CSR_CONDX | FPU_CSR_FS; c->options = MIPS_CPU_TLB | MIPS_CPU_3K_CACHE | MIPS_CPU_NOFPUEX; if (__cpu_has_fpu()) @@ -353,7 +1070,8 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) break; case PRID_IMP_R4000: if (read_c0_config() & CONF_SC) { - if ((c->processor_id & 0xff) >= PRID_REV_R4400) { + if ((c->processor_id & PRID_REV_MASK) >= + PRID_REV_R4400) { c->cputype = CPU_R4400PC; __cpu_name[cpu] = "R4400PC"; } else { @@ -361,64 +1079,48 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) __cpu_name[cpu] = "R4000PC"; } } else { - if ((c->processor_id & 0xff) >= PRID_REV_R4400) { - c->cputype = CPU_R4400SC; - __cpu_name[cpu] = "R4400SC"; + int cca = read_c0_config() & CONF_CM_CMASK; + int mc; + + /* + * SC and MC versions can't be reliably told apart, + * but only the latter support coherent caching + * modes so assume the firmware has set the KSEG0 + * coherency attribute reasonably (if uncached, we + * assume SC). + */ + switch (cca) { + case CONF_CM_CACHABLE_CE: + case CONF_CM_CACHABLE_COW: + case CONF_CM_CACHABLE_CUW: + mc = 1; + break; + default: + mc = 0; + break; + } + if ((c->processor_id & PRID_REV_MASK) >= + PRID_REV_R4400) { + c->cputype = mc ? CPU_R4400MC : CPU_R4400SC; + __cpu_name[cpu] = mc ? "R4400MC" : "R4400SC"; } else { - c->cputype = CPU_R4000SC; - __cpu_name[cpu] = "R4000SC"; + c->cputype = mc ? CPU_R4000MC : CPU_R4000SC; + __cpu_name[cpu] = mc ? "R4000MC" : "R4000SC"; } } set_isa(c, MIPS_CPU_ISA_III); + c->fpu_msk31 |= FPU_CSR_CONDX; c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | MIPS_CPU_WATCH | MIPS_CPU_VCE | MIPS_CPU_LLSC; c->tlbsize = 48; break; - case PRID_IMP_VR41XX: - set_isa(c, MIPS_CPU_ISA_III); - c->options = R4K_OPTS; - c->tlbsize = 32; - switch (c->processor_id & 0xf0) { - case PRID_REV_VR4111: - c->cputype = CPU_VR4111; - __cpu_name[cpu] = "NEC VR4111"; - break; - case PRID_REV_VR4121: - c->cputype = CPU_VR4121; - __cpu_name[cpu] = "NEC VR4121"; - break; - case PRID_REV_VR4122: - if ((c->processor_id & 0xf) < 0x3) { - c->cputype = CPU_VR4122; - __cpu_name[cpu] = "NEC VR4122"; - } else { - c->cputype = CPU_VR4181A; - __cpu_name[cpu] = "NEC VR4181A"; - } - break; - case PRID_REV_VR4130: - if ((c->processor_id & 0xf) < 0x4) { - c->cputype = CPU_VR4131; - __cpu_name[cpu] = "NEC VR4131"; - } else { - c->cputype = CPU_VR4133; - c->options |= MIPS_CPU_LLSC; - __cpu_name[cpu] = "NEC VR4133"; - } - break; - default: - printk(KERN_INFO "Unexpected CPU of NEC VR4100 series\n"); - c->cputype = CPU_VR41XX; - __cpu_name[cpu] = "NEC Vr41xx"; - break; - } - break; case PRID_IMP_R4300: c->cputype = CPU_R4300; __cpu_name[cpu] = "R4300"; set_isa(c, MIPS_CPU_ISA_III); + c->fpu_msk31 |= FPU_CSR_CONDX; c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | MIPS_CPU_LLSC; c->tlbsize = 32; @@ -427,6 +1129,7 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) c->cputype = CPU_R4600; __cpu_name[cpu] = "R4600"; set_isa(c, MIPS_CPU_ISA_III); + c->fpu_msk31 |= FPU_CSR_CONDX; c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | MIPS_CPU_LLSC; c->tlbsize = 48; @@ -437,41 +1140,21 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) * This processor doesn't have an MMU, so it's not * "real easy" to run Linux on it. It is left purely * for documentation. Commented out because it shares - * it's c0_prid id number with the TX3900. + * its c0_prid id number with the TX3900. */ c->cputype = CPU_R4650; __cpu_name[cpu] = "R4650"; set_isa(c, MIPS_CPU_ISA_III); + c->fpu_msk31 |= FPU_CSR_CONDX; c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_LLSC; c->tlbsize = 48; break; #endif - case PRID_IMP_TX39: - c->options = MIPS_CPU_TLB | MIPS_CPU_TX39_CACHE; - - if ((c->processor_id & 0xf0) == (PRID_REV_TX3927 & 0xf0)) { - c->cputype = CPU_TX3927; - __cpu_name[cpu] = "TX3927"; - c->tlbsize = 64; - } else { - switch (c->processor_id & 0xff) { - case PRID_REV_TX3912: - c->cputype = CPU_TX3912; - __cpu_name[cpu] = "TX3912"; - c->tlbsize = 32; - break; - case PRID_REV_TX3922: - c->cputype = CPU_TX3922; - __cpu_name[cpu] = "TX3922"; - c->tlbsize = 64; - break; - } - } - break; case PRID_IMP_R4700: c->cputype = CPU_R4700; __cpu_name[cpu] = "R4700"; set_isa(c, MIPS_CPU_ISA_III); + c->fpu_msk31 |= FPU_CSR_CONDX; c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | MIPS_CPU_LLSC; c->tlbsize = 48; @@ -480,6 +1163,7 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) c->cputype = CPU_TX49XX; __cpu_name[cpu] = "R49XX"; set_isa(c, MIPS_CPU_ISA_III); + c->fpu_msk31 |= FPU_CSR_CONDX; c->options = R4K_OPTS | MIPS_CPU_LLSC; if (!(c->processor_id & 0x08)) c->options |= MIPS_CPU_FPU | MIPS_CPU_32FPR; @@ -493,14 +1177,6 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) MIPS_CPU_LLSC; c->tlbsize = 48; break; - case PRID_IMP_R5432: - c->cputype = CPU_R5432; - __cpu_name[cpu] = "R5432"; - set_isa(c, MIPS_CPU_ISA_IV); - c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | - MIPS_CPU_WATCH | MIPS_CPU_LLSC; - c->tlbsize = 48; - break; case PRID_IMP_R5500: c->cputype = CPU_R5500; __cpu_name[cpu] = "R5500"; @@ -517,22 +1193,6 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) MIPS_CPU_DIVEC | MIPS_CPU_LLSC; c->tlbsize = 48; break; - case PRID_IMP_R6000: - c->cputype = CPU_R6000; - __cpu_name[cpu] = "R6000"; - set_isa(c, MIPS_CPU_ISA_II); - c->options = MIPS_CPU_TLB | MIPS_CPU_FPU | - MIPS_CPU_LLSC; - c->tlbsize = 32; - break; - case PRID_IMP_R6000A: - c->cputype = CPU_R6000A; - __cpu_name[cpu] = "R6000A"; - set_isa(c, MIPS_CPU_ISA_II); - c->options = MIPS_CPU_TLB | MIPS_CPU_FPU | - MIPS_CPU_LLSC; - c->tlbsize = 32; - break; case PRID_IMP_RM7000: c->cputype = CPU_RM7000; __cpu_name[cpu] = "RM7000"; @@ -549,30 +1209,6 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) */ c->tlbsize = (read_c0_info() & (1 << 29)) ? 64 : 48; break; - case PRID_IMP_RM9000: - c->cputype = CPU_RM9000; - __cpu_name[cpu] = "RM9000"; - set_isa(c, MIPS_CPU_ISA_IV); - c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | - MIPS_CPU_LLSC; - /* - * Bit 29 in the info register of the RM9000 - * indicates if the TLB has 48 or 64 entries. - * - * 29 1 => 64 entry JTLB - * 0 => 48 entry JTLB - */ - c->tlbsize = (read_c0_info() & (1 << 29)) ? 64 : 48; - break; - case PRID_IMP_R8000: - c->cputype = CPU_R8000; - __cpu_name[cpu] = "RM8000"; - set_isa(c, MIPS_CPU_ISA_IV); - c->options = MIPS_CPU_TLB | MIPS_CPU_4KEX | - MIPS_CPU_FPU | MIPS_CPU_32FPR | - MIPS_CPU_LLSC; - c->tlbsize = 384; /* has weird TLB: 3-way x 128 */ - break; case PRID_IMP_R10000: c->cputype = CPU_R10000; __cpu_name[cpu] = "R10000"; @@ -592,44 +1228,74 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) MIPS_CPU_COUNTER | MIPS_CPU_WATCH | MIPS_CPU_LLSC; c->tlbsize = 64; + write_c0_r10k_diag(read_c0_r10k_diag() | R10K_DIAG_E_GHIST); break; case PRID_IMP_R14000: - c->cputype = CPU_R14000; - __cpu_name[cpu] = "R14000"; + if (((c->processor_id >> 4) & 0x0f) > 2) { + c->cputype = CPU_R16000; + __cpu_name[cpu] = "R16000"; + } else { + c->cputype = CPU_R14000; + __cpu_name[cpu] = "R14000"; + } set_isa(c, MIPS_CPU_ISA_IV); c->options = MIPS_CPU_TLB | MIPS_CPU_4K_CACHE | MIPS_CPU_4KEX | MIPS_CPU_FPU | MIPS_CPU_32FPR | MIPS_CPU_COUNTER | MIPS_CPU_WATCH | MIPS_CPU_LLSC; c->tlbsize = 64; + write_c0_r10k_diag(read_c0_r10k_diag() | R10K_DIAG_E_GHIST); break; - case PRID_IMP_LOONGSON2: - c->cputype = CPU_LOONGSON2; - __cpu_name[cpu] = "ICT Loongson-2"; - + case PRID_IMP_LOONGSON_64C: /* Loongson-2/3 */ switch (c->processor_id & PRID_REV_MASK) { case PRID_REV_LOONGSON2E: + c->cputype = CPU_LOONGSON2EF; + __cpu_name[cpu] = "ICT Loongson-2"; set_elf_platform(cpu, "loongson2e"); + set_isa(c, MIPS_CPU_ISA_III); + c->fpu_msk31 |= FPU_CSR_CONDX; break; case PRID_REV_LOONGSON2F: + c->cputype = CPU_LOONGSON2EF; + __cpu_name[cpu] = "ICT Loongson-2"; set_elf_platform(cpu, "loongson2f"); + set_isa(c, MIPS_CPU_ISA_III); + c->fpu_msk31 |= FPU_CSR_CONDX; + break; + case PRID_REV_LOONGSON3A_R1: + c->cputype = CPU_LOONGSON64; + __cpu_name[cpu] = "ICT Loongson-3"; + set_elf_platform(cpu, "loongson3a"); + set_isa(c, MIPS_CPU_ISA_M64R1); + c->ases |= (MIPS_ASE_LOONGSON_MMI | MIPS_ASE_LOONGSON_CAM | + MIPS_ASE_LOONGSON_EXT); + break; + case PRID_REV_LOONGSON3B_R1: + case PRID_REV_LOONGSON3B_R2: + c->cputype = CPU_LOONGSON64; + __cpu_name[cpu] = "ICT Loongson-3"; + set_elf_platform(cpu, "loongson3b"); + set_isa(c, MIPS_CPU_ISA_M64R1); + c->ases |= (MIPS_ASE_LOONGSON_MMI | MIPS_ASE_LOONGSON_CAM | + MIPS_ASE_LOONGSON_EXT); break; } - set_isa(c, MIPS_CPU_ISA_III); c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_LLSC | MIPS_CPU_32FPR; c->tlbsize = 64; + set_cpu_asid_mask(c, MIPS_ENTRYHI_ASID); + c->writecombine = _CACHE_UNCACHED_ACCELERATED; break; - case PRID_IMP_LOONGSON1: + case PRID_IMP_LOONGSON_32: decode_configs(c); - c->cputype = CPU_LOONGSON1; + c->cputype = CPU_LOONGSON32; switch (c->processor_id & PRID_REV_MASK) { - case PRID_REV_LOONGSON1B: - __cpu_name[cpu] = "Loongson 1B"; + case PRID_REV_LOONGSON1: + __cpu_name[cpu] = "ICT Loongson-1"; break; } @@ -639,79 +1305,175 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu) { - decode_configs(c); - switch (c->processor_id & 0xff00) { + c->writecombine = _CACHE_UNCACHED_ACCELERATED; + switch (c->processor_id & PRID_IMP_MASK) { + case PRID_IMP_QEMU_GENERIC: + c->writecombine = _CACHE_UNCACHED; + c->cputype = CPU_QEMU_GENERIC; + __cpu_name[cpu] = "MIPS GENERIC QEMU"; + break; case PRID_IMP_4KC: c->cputype = CPU_4KC; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS 4Kc"; break; case PRID_IMP_4KEC: case PRID_IMP_4KECR2: c->cputype = CPU_4KEC; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS 4KEc"; break; case PRID_IMP_4KSC: case PRID_IMP_4KSD: c->cputype = CPU_4KSC; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS 4KSc"; break; case PRID_IMP_5KC: c->cputype = CPU_5KC; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS 5Kc"; break; case PRID_IMP_5KE: c->cputype = CPU_5KE; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS 5KE"; break; case PRID_IMP_20KC: c->cputype = CPU_20KC; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS 20Kc"; break; case PRID_IMP_24K: c->cputype = CPU_24K; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS 24Kc"; break; case PRID_IMP_24KE: c->cputype = CPU_24K; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS 24KEc"; break; case PRID_IMP_25KF: c->cputype = CPU_25KF; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS 25Kc"; break; case PRID_IMP_34K: c->cputype = CPU_34K; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS 34Kc"; + cpu_set_mt_per_tc_perf(c); break; case PRID_IMP_74K: c->cputype = CPU_74K; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS 74Kc"; break; case PRID_IMP_M14KC: c->cputype = CPU_M14KC; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS M14Kc"; break; case PRID_IMP_M14KEC: c->cputype = CPU_M14KEC; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS M14KEc"; break; case PRID_IMP_1004K: c->cputype = CPU_1004K; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS 1004Kc"; + cpu_set_mt_per_tc_perf(c); break; case PRID_IMP_1074K: - c->cputype = CPU_74K; + c->cputype = CPU_1074K; + c->writecombine = _CACHE_UNCACHED; __cpu_name[cpu] = "MIPS 1074Kc"; break; + case PRID_IMP_INTERAPTIV_UP: + c->cputype = CPU_INTERAPTIV; + __cpu_name[cpu] = "MIPS interAptiv"; + cpu_set_mt_per_tc_perf(c); + break; + case PRID_IMP_INTERAPTIV_MP: + c->cputype = CPU_INTERAPTIV; + __cpu_name[cpu] = "MIPS interAptiv (multi)"; + cpu_set_mt_per_tc_perf(c); + break; + case PRID_IMP_PROAPTIV_UP: + c->cputype = CPU_PROAPTIV; + __cpu_name[cpu] = "MIPS proAptiv"; + break; + case PRID_IMP_PROAPTIV_MP: + c->cputype = CPU_PROAPTIV; + __cpu_name[cpu] = "MIPS proAptiv (multi)"; + break; + case PRID_IMP_P5600: + c->cputype = CPU_P5600; + __cpu_name[cpu] = "MIPS P5600"; + break; + case PRID_IMP_P6600: + c->cputype = CPU_P6600; + __cpu_name[cpu] = "MIPS P6600"; + break; + case PRID_IMP_I6400: + c->cputype = CPU_I6400; + __cpu_name[cpu] = "MIPS I6400"; + break; + case PRID_IMP_I6500: + c->cputype = CPU_I6500; + __cpu_name[cpu] = "MIPS I6500"; + break; + case PRID_IMP_M5150: + c->cputype = CPU_M5150; + __cpu_name[cpu] = "MIPS M5150"; + break; + case PRID_IMP_M6250: + c->cputype = CPU_M6250; + __cpu_name[cpu] = "MIPS M6250"; + break; } + decode_configs(c); + spram_config(); + + mm_config(c); + + switch (__get_cpu_type(c->cputype)) { + case CPU_M5150: + case CPU_P5600: + set_isa(c, MIPS_CPU_ISA_M32R5); + break; + case CPU_I6500: + c->options |= MIPS_CPU_SHARED_FTLB_ENTRIES; + fallthrough; + case CPU_I6400: + c->options |= MIPS_CPU_SHARED_FTLB_RAM; + fallthrough; + default: + break; + } + + /* Recent MIPS cores use the implementation-dependent ExcCode 16 for + * cache/FTLB parity exceptions. + */ + switch (__get_cpu_type(c->cputype)) { + case CPU_PROAPTIV: + case CPU_P5600: + case CPU_P6600: + case CPU_I6400: + case CPU_I6500: + c->options |= MIPS_CPU_FTLBPAREX; + break; + } } static inline void cpu_probe_alchemy(struct cpuinfo_mips *c, unsigned int cpu) { decode_configs(c); - switch (c->processor_id & 0xff00) { + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_AU1_REV1: case PRID_IMP_AU1_REV2: c->cputype = CPU_ALCHEMY; @@ -730,7 +1492,7 @@ static inline void cpu_probe_alchemy(struct cpuinfo_mips *c, unsigned int cpu) break; case 4: __cpu_name[cpu] = "Au1200"; - if ((c->processor_id & 0xff) == 2) + if ((c->processor_id & PRID_REV_MASK) == 2) __cpu_name[cpu] = "Au1250"; break; case 5: @@ -741,6 +1503,10 @@ static inline void cpu_probe_alchemy(struct cpuinfo_mips *c, unsigned int cpu) break; } break; + case PRID_IMP_NETLOGIC_AU13XX: + c->cputype = CPU_ALCHEMY; + __cpu_name[cpu] = "Au1300"; + break; } } @@ -748,12 +1514,13 @@ static inline void cpu_probe_sibyte(struct cpuinfo_mips *c, unsigned int cpu) { decode_configs(c); - switch (c->processor_id & 0xff00) { + c->writecombine = _CACHE_UNCACHED_ACCELERATED; + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_SB1: c->cputype = CPU_SB1; __cpu_name[cpu] = "SiByte SB1"; /* FPU in pass1 is known to have issues. */ - if ((c->processor_id & 0xff) < 0x02) + if ((c->processor_id & PRID_REV_MASK) < 0x02) c->options &= ~(MIPS_CPU_FPU | MIPS_CPU_32FPR); break; case PRID_IMP_SB1A: @@ -766,7 +1533,7 @@ static inline void cpu_probe_sibyte(struct cpuinfo_mips *c, unsigned int cpu) static inline void cpu_probe_sandcraft(struct cpuinfo_mips *c, unsigned int cpu) { decode_configs(c); - switch (c->processor_id & 0xff00) { + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_SR71000: c->cputype = CPU_SR71000; __cpu_name[cpu] = "Sandcraft SR71000"; @@ -779,7 +1546,7 @@ static inline void cpu_probe_sandcraft(struct cpuinfo_mips *c, unsigned int cpu) static inline void cpu_probe_nxp(struct cpuinfo_mips *c, unsigned int cpu) { decode_configs(c); - switch (c->processor_id & 0xff00) { + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_PR4450: c->cputype = CPU_PR4450; __cpu_name[cpu] = "Philips PR4450"; @@ -791,7 +1558,7 @@ static inline void cpu_probe_nxp(struct cpuinfo_mips *c, unsigned int cpu) static inline void cpu_probe_broadcom(struct cpuinfo_mips *c, unsigned int cpu) { decode_configs(c); - switch (c->processor_id & 0xff00) { + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_BMIPS32_REV4: case PRID_IMP_BMIPS32_REV8: c->cputype = CPU_BMIPS32; @@ -804,15 +1571,18 @@ static inline void cpu_probe_broadcom(struct cpuinfo_mips *c, unsigned int cpu) c->cputype = CPU_BMIPS3300; __cpu_name[cpu] = "Broadcom BMIPS3300"; set_elf_platform(cpu, "bmips3300"); + reserve_exception_space(0x400, VECTORSPACING * 64); break; case PRID_IMP_BMIPS43XX: { - int rev = c->processor_id & 0xff; + int rev = c->processor_id & PRID_REV_MASK; if (rev >= PRID_REV_BMIPS4380_LO && rev <= PRID_REV_BMIPS4380_HI) { c->cputype = CPU_BMIPS4380; __cpu_name[cpu] = "Broadcom BMIPS4380"; set_elf_platform(cpu, "bmips4380"); + c->options |= MIPS_CPU_RIXI; + reserve_exception_space(0x400, VECTORSPACING * 64); } else { c->cputype = CPU_BMIPS4350; __cpu_name[cpu] = "Broadcom BMIPS4350"; @@ -821,10 +1591,15 @@ static inline void cpu_probe_broadcom(struct cpuinfo_mips *c, unsigned int cpu) break; } case PRID_IMP_BMIPS5000: + case PRID_IMP_BMIPS5200: c->cputype = CPU_BMIPS5000; - __cpu_name[cpu] = "Broadcom BMIPS5000"; + if ((c->processor_id & PRID_IMP_MASK) == PRID_IMP_BMIPS5200) + __cpu_name[cpu] = "Broadcom BMIPS5200"; + else + __cpu_name[cpu] = "Broadcom BMIPS5000"; set_elf_platform(cpu, "bmips5000"); - c->options |= MIPS_CPU_ULRI; + c->options |= MIPS_CPU_ULRI | MIPS_CPU_RIXI; + reserve_exception_space(0x1000, VECTORSPACING * 64); break; } } @@ -832,7 +1607,9 @@ static inline void cpu_probe_broadcom(struct cpuinfo_mips *c, unsigned int cpu) static inline void cpu_probe_cavium(struct cpuinfo_mips *c, unsigned int cpu) { decode_configs(c); - switch (c->processor_id & 0xff00) { + /* Octeon has different cache interface */ + c->options &= ~MIPS_CPU_4K_CACHE; + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_CAVIUM_CN38XX: case PRID_IMP_CAVIUM_CN31XX: case PRID_IMP_CAVIUM_CN30XX: @@ -852,10 +1629,19 @@ platform: case PRID_IMP_CAVIUM_CN63XX: case PRID_IMP_CAVIUM_CN66XX: case PRID_IMP_CAVIUM_CN68XX: + case PRID_IMP_CAVIUM_CNF71XX: c->cputype = CPU_CAVIUM_OCTEON2; __cpu_name[cpu] = "Cavium Octeon II"; set_elf_platform(cpu, "octeon2"); break; + case PRID_IMP_CAVIUM_CN70XX: + case PRID_IMP_CAVIUM_CN73XX: + case PRID_IMP_CAVIUM_CNF75XX: + case PRID_IMP_CAVIUM_CN78XX: + c->cputype = CPU_CAVIUM_OCTEON3; + __cpu_name[cpu] = "Cavium Octeon III"; + set_elf_platform(cpu, "octeon3"); + break; default: printk(KERN_INFO "Unknown Octeon chip!\n"); c->cputype = CPU_UNKNOWN; @@ -863,94 +1649,186 @@ platform: } } -static inline void cpu_probe_ingenic(struct cpuinfo_mips *c, unsigned int cpu) +#ifdef CONFIG_CPU_LOONGSON64 +#include <loongson_regs.h> + +static inline void decode_cpucfg(struct cpuinfo_mips *c) +{ + u32 cfg1 = read_cpucfg(LOONGSON_CFG1); + u32 cfg2 = read_cpucfg(LOONGSON_CFG2); + u32 cfg3 = read_cpucfg(LOONGSON_CFG3); + + if (cfg1 & LOONGSON_CFG1_MMI) + c->ases |= MIPS_ASE_LOONGSON_MMI; + + if (cfg2 & LOONGSON_CFG2_LEXT1) + c->ases |= MIPS_ASE_LOONGSON_EXT; + + if (cfg2 & LOONGSON_CFG2_LEXT2) + c->ases |= MIPS_ASE_LOONGSON_EXT2; + + if (cfg2 & LOONGSON_CFG2_LSPW) { + c->options |= MIPS_CPU_LDPTE; + c->guest.options |= MIPS_CPU_LDPTE; + } + + if (cfg3 & LOONGSON_CFG3_LCAMP) + c->ases |= MIPS_ASE_LOONGSON_CAM; +} + +static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) { + c->cputype = CPU_LOONGSON64; + + /* All Loongson processors covered here define ExcCode 16 as GSExc. */ decode_configs(c); - /* JZRISC does not implement the CP0 counter. */ - c->options &= ~MIPS_CPU_COUNTER; - switch (c->processor_id & 0xff00) { - case PRID_IMP_JZRISC: - c->cputype = CPU_JZRISC; - __cpu_name[cpu] = "Ingenic JZRISC"; + c->options |= MIPS_CPU_GSEXCEX; + + switch (c->processor_id & PRID_IMP_MASK) { + case PRID_IMP_LOONGSON_64R: /* Loongson-64 Reduced */ + switch (c->processor_id & PRID_REV_MASK) { + case PRID_REV_LOONGSON2K_R1_0: + case PRID_REV_LOONGSON2K_R1_1: + case PRID_REV_LOONGSON2K_R1_2: + case PRID_REV_LOONGSON2K_R1_3: + __cpu_name[cpu] = "Loongson-2K"; + set_elf_platform(cpu, "gs264e"); + set_isa(c, MIPS_CPU_ISA_M64R2); + break; + } + c->ases |= (MIPS_ASE_LOONGSON_MMI | MIPS_ASE_LOONGSON_EXT | + MIPS_ASE_LOONGSON_EXT2); + break; + case PRID_IMP_LOONGSON_64C: /* Loongson-3 Classic */ + switch (c->processor_id & PRID_REV_MASK) { + case PRID_REV_LOONGSON3A_R2_0: + case PRID_REV_LOONGSON3A_R2_1: + __cpu_name[cpu] = "ICT Loongson-3"; + set_elf_platform(cpu, "loongson3a"); + set_isa(c, MIPS_CPU_ISA_M64R2); + break; + case PRID_REV_LOONGSON3A_R3_0: + case PRID_REV_LOONGSON3A_R3_1: + __cpu_name[cpu] = "ICT Loongson-3"; + set_elf_platform(cpu, "loongson3a"); + set_isa(c, MIPS_CPU_ISA_M64R2); + break; + } + /* + * Loongson-3 Classic did not implement MIPS standard TLBINV + * but implemented TLBINVF and EHINV. As currently we're only + * using these two features, enable MIPS_CPU_TLBINV as well. + * + * Also some early Loongson-3A2000 had wrong TLB type in Config + * register, we correct it here. + */ + c->options |= MIPS_CPU_FTLB | MIPS_CPU_TLBINV | MIPS_CPU_LDPTE; + c->ases |= (MIPS_ASE_LOONGSON_MMI | MIPS_ASE_LOONGSON_CAM | + MIPS_ASE_LOONGSON_EXT | MIPS_ASE_LOONGSON_EXT2); + c->ases &= ~MIPS_ASE_VZ; /* VZ of Loongson-3A2000/3000 is incomplete */ + change_c0_config6(LOONGSON_CONF6_EXTIMER | LOONGSON_CONF6_INTIMER, + LOONGSON_CONF6_INTIMER); + break; + case PRID_IMP_LOONGSON_64G: + __cpu_name[cpu] = "ICT Loongson-3"; + set_elf_platform(cpu, "loongson3a"); + set_isa(c, MIPS_CPU_ISA_M64R2); + decode_cpucfg(c); + change_c0_config6(LOONGSON_CONF6_EXTIMER | LOONGSON_CONF6_INTIMER, + LOONGSON_CONF6_INTIMER); break; default: - panic("Unknown Ingenic Processor ID!"); + panic("Unknown Loongson Processor ID!"); break; } } +#else +static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) { } +#endif -static inline void cpu_probe_netlogic(struct cpuinfo_mips *c, int cpu) +static inline void cpu_probe_ingenic(struct cpuinfo_mips *c, unsigned int cpu) { decode_configs(c); - if ((c->processor_id & 0xff00) == PRID_IMP_NETLOGIC_AU13XX) { - c->cputype = CPU_ALCHEMY; - __cpu_name[cpu] = "Au1300"; - /* following stuff is not for Alchemy */ - return; - } + /* + * XBurst misses a config2 register, so config3 decode was skipped in + * decode_configs(). + */ + decode_config3(c); + + /* XBurst does not implement the CP0 counter. */ + c->options &= ~MIPS_CPU_COUNTER; + BUG_ON(__builtin_constant_p(cpu_has_counter) && cpu_has_counter); + + /* XBurst has virtually tagged icache */ + c->icache.flags |= MIPS_CACHE_VTAG; + + switch (c->processor_id & PRID_IMP_MASK) { + + /* XBurst®1 with MXU1.0/MXU1.1 SIMD ISA */ + case PRID_IMP_XBURST_REV1: + + /* + * The XBurst core by default attempts to avoid branch target + * buffer lookups by detecting & special casing loops. This + * feature will cause BogoMIPS and lpj calculate in error. + * Set cp0 config7 bit 4 to disable this feature. + */ + set_c0_config7(MIPS_CONF7_BTB_LOOP_EN); + + switch (c->processor_id & PRID_COMP_MASK) { + + /* + * The config0 register in the XBurst CPUs with a processor ID of + * PRID_COMP_INGENIC_D0 report themselves as MIPS32r2 compatible, + * but they don't actually support this ISA. + */ + case PRID_COMP_INGENIC_D0: + c->isa_level &= ~MIPS_CPU_ISA_M32R2; + + /* FPU is not properly detected on JZ4760(B). */ + if (c->processor_id == 0x2ed0024f) + c->options |= MIPS_CPU_FPU; + + fallthrough; + + /* + * The config0 register in the XBurst CPUs with a processor ID of + * PRID_COMP_INGENIC_D0 or PRID_COMP_INGENIC_D1 has an abandoned + * huge page tlb mode, this mode is not compatible with the MIPS + * standard, it will cause tlbmiss and into an infinite loop + * (line 21 in the tlb-funcs.S) when starting the init process. + * After chip reset, the default is HPTLB mode, Write 0xa9000000 + * to cp0 register 5 sel 4 to switch back to VTLB mode to prevent + * getting stuck. + */ + case PRID_COMP_INGENIC_D1: + write_c0_page_ctrl(XBURST_PAGECTRL_HPTLB_DIS); + break; + + default: + break; + } + fallthrough; - c->options = (MIPS_CPU_TLB | - MIPS_CPU_4KEX | - MIPS_CPU_COUNTER | - MIPS_CPU_DIVEC | - MIPS_CPU_WATCH | - MIPS_CPU_EJTAG | - MIPS_CPU_LLSC); - - switch (c->processor_id & 0xff00) { - case PRID_IMP_NETLOGIC_XLP8XX: - case PRID_IMP_NETLOGIC_XLP3XX: - c->cputype = CPU_XLP; - __cpu_name[cpu] = "Netlogic XLP"; - break; - - case PRID_IMP_NETLOGIC_XLR732: - case PRID_IMP_NETLOGIC_XLR716: - case PRID_IMP_NETLOGIC_XLR532: - case PRID_IMP_NETLOGIC_XLR308: - case PRID_IMP_NETLOGIC_XLR532C: - case PRID_IMP_NETLOGIC_XLR516C: - case PRID_IMP_NETLOGIC_XLR508C: - case PRID_IMP_NETLOGIC_XLR308C: - c->cputype = CPU_XLR; - __cpu_name[cpu] = "Netlogic XLR"; - break; - - case PRID_IMP_NETLOGIC_XLS608: - case PRID_IMP_NETLOGIC_XLS408: - case PRID_IMP_NETLOGIC_XLS404: - case PRID_IMP_NETLOGIC_XLS208: - case PRID_IMP_NETLOGIC_XLS204: - case PRID_IMP_NETLOGIC_XLS108: - case PRID_IMP_NETLOGIC_XLS104: - case PRID_IMP_NETLOGIC_XLS616B: - case PRID_IMP_NETLOGIC_XLS608B: - case PRID_IMP_NETLOGIC_XLS416B: - case PRID_IMP_NETLOGIC_XLS412B: - case PRID_IMP_NETLOGIC_XLS408B: - case PRID_IMP_NETLOGIC_XLS404B: - c->cputype = CPU_XLR; - __cpu_name[cpu] = "Netlogic XLS"; + /* XBurst®1 with MXU2.0 SIMD ISA */ + case PRID_IMP_XBURST_REV2: + /* Ingenic uses the WA bit to achieve write-combine memory writes */ + c->writecombine = _CACHE_CACHABLE_WA; + c->cputype = CPU_XBURST; + __cpu_name[cpu] = "Ingenic XBurst"; break; - default: - pr_info("Unknown Netlogic chip id [%02x]!\n", - c->processor_id); - c->cputype = CPU_XLR; + /* XBurst®2 with MXU2.1 SIMD ISA */ + case PRID_IMP_XBURST2: + c->cputype = CPU_XBURST; + __cpu_name[cpu] = "Ingenic XBurst II"; break; - } - if (c->cputype == CPU_XLP) { - set_isa(c, MIPS_CPU_ISA_M64R2); - c->options |= (MIPS_CPU_FPU | MIPS_CPU_ULRI | MIPS_CPU_MCHECK); - /* This will be updated again after all threads are woken up */ - c->tlbsize = ((read_c0_config6() >> 16) & 0xffff) + 1; - } else { - set_isa(c, MIPS_CPU_ISA_M64R1); - c->tlbsize = ((read_c0_config1() >> 25) & 0x3f) + 1; + default: + panic("Unknown Ingenic Processor ID!"); + break; } - c->kscratch_mask = 0xf; } #ifdef CONFIG_64BIT @@ -961,18 +1839,29 @@ EXPORT_SYMBOL(__ua_limit); const char *__cpu_name[NR_CPUS]; const char *__elf_platform; +const char *__elf_base_platform; -__cpuinit void cpu_probe(void) +void cpu_probe(void) { struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int cpu = smp_processor_id(); + /* + * Set a default elf platform, cpu probe may later + * overwrite it with a more precise value + */ + set_elf_platform(cpu, "mips"); + c->processor_id = PRID_IMP_UNKNOWN; c->fpu_id = FPIR_IMP_NONE; c->cputype = CPU_UNKNOWN; + c->writecombine = _CACHE_UNCACHED; + + c->fpu_csr31 = FPU_CSR_RN; + c->fpu_msk31 = FPU_CSR_RSVD | FPU_CSR_ABS2008 | FPU_CSR_NAN2008; c->processor_id = read_c0_prid(); - switch (c->processor_id & 0xff0000) { + switch (c->processor_id & PRID_COMP_MASK) { case PRID_COMP_LEGACY: cpu_probe_legacy(c, cpu); break; @@ -980,6 +1869,7 @@ __cpuinit void cpu_probe(void) cpu_probe_mips(c, cpu); break; case PRID_COMP_ALCHEMY: + case PRID_COMP_NETLOGIC: cpu_probe_alchemy(c, cpu); break; case PRID_COMP_SIBYTE: @@ -997,11 +1887,14 @@ __cpuinit void cpu_probe(void) case PRID_COMP_CAVIUM: cpu_probe_cavium(c, cpu); break; - case PRID_COMP_INGENIC: - cpu_probe_ingenic(c, cpu); + case PRID_COMP_LOONGSON: + cpu_probe_loongson(c, cpu); break; - case PRID_COMP_NETLOGIC: - cpu_probe_netlogic(c, cpu); + case PRID_COMP_INGENIC_13: + case PRID_COMP_INGENIC_D0: + case PRID_COMP_INGENIC_D1: + case PRID_COMP_INGENIC_E1: + cpu_probe_ingenic(c, cpu); break; } @@ -1015,23 +1908,33 @@ __cpuinit void cpu_probe(void) */ BUG_ON(current_cpu_type() != c->cputype); + if (cpu_has_rixi) { + /* Enable the RIXI exceptions */ + set_c0_pagegrain(PG_IEC); + back_to_back_c0_hazard(); + /* Verify the IEC bit is set */ + if (read_c0_pagegrain() & PG_IEC) + c->options |= MIPS_CPU_RIXIEX; + } + if (mips_fpu_disabled) c->options &= ~MIPS_CPU_FPU; if (mips_dsp_disabled) c->ases &= ~(MIPS_ASE_DSP | MIPS_ASE_DSP2P); - if (c->options & MIPS_CPU_FPU) { - c->fpu_id = cpu_get_fpu_id(); - - if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M32R2 | - MIPS_CPU_ISA_M64R1 | MIPS_CPU_ISA_M64R2)) { - if (c->fpu_id & MIPS_FPIR_3D) - c->ases |= MIPS_ASE_MIPS3D; - } + if (mips_htw_disabled) { + c->options &= ~MIPS_CPU_HTW; + write_c0_pwctl(read_c0_pwctl() & + ~(1 << MIPS_PWCTL_PWEN_SHIFT)); } - if (cpu_has_mips_r2) { + if (c->options & MIPS_CPU_FPU) + cpu_set_fpu_opts(c); + else + cpu_set_nofpu_opts(c); + + if (cpu_has_mips_r2_r6) { c->srsets = ((read_c0_srsctl() >> 26) & 0x0f) + 1; /* R2 has Performance Counter Interrupt indicator */ c->options |= MIPS_CPU_PCI; @@ -1039,20 +1942,145 @@ __cpuinit void cpu_probe(void) else c->srsets = 1; + if (cpu_has_mips_r6) + elf_hwcap |= HWCAP_MIPS_R6; + + if (cpu_has_msa) { + c->msa_id = cpu_get_msa_id(); + WARN(c->msa_id & MSA_IR_WRPF, + "Vector register partitioning unimplemented!"); + elf_hwcap |= HWCAP_MIPS_MSA; + } + + if (cpu_has_mips16) + elf_hwcap |= HWCAP_MIPS_MIPS16; + + if (cpu_has_mdmx) + elf_hwcap |= HWCAP_MIPS_MDMX; + + if (cpu_has_mips3d) + elf_hwcap |= HWCAP_MIPS_MIPS3D; + + if (cpu_has_smartmips) + elf_hwcap |= HWCAP_MIPS_SMARTMIPS; + + if (cpu_has_dsp) + elf_hwcap |= HWCAP_MIPS_DSP; + + if (cpu_has_dsp2) + elf_hwcap |= HWCAP_MIPS_DSP2; + + if (cpu_has_dsp3) + elf_hwcap |= HWCAP_MIPS_DSP3; + + if (cpu_has_mips16e2) + elf_hwcap |= HWCAP_MIPS_MIPS16E2; + + if (cpu_has_loongson_mmi) + elf_hwcap |= HWCAP_LOONGSON_MMI; + + if (cpu_has_loongson_ext) + elf_hwcap |= HWCAP_LOONGSON_EXT; + + if (cpu_has_loongson_ext2) + elf_hwcap |= HWCAP_LOONGSON_EXT2; + + if (cpu_has_vz) + cpu_probe_vz(c); + cpu_probe_vmbits(c); + /* Synthesize CPUCFG data if running on Loongson processors; + * no-op otherwise. + * + * This looks at previously probed features, so keep this at bottom. + */ + loongson3_cpucfg_synthesize_data(c); + #ifdef CONFIG_64BIT if (cpu == 0) __ua_limit = ~((1ull << cpu_vmbits) - 1); #endif + + reserve_exception_space(0, 0x1000); } -__cpuinit void cpu_report(void) +void cpu_report(void) { struct cpuinfo_mips *c = ¤t_cpu_data; - printk(KERN_INFO "CPU revision is: %08x (%s)\n", - c->processor_id, cpu_name_string()); + pr_info("CPU%d revision is: %08x (%s)\n", + smp_processor_id(), c->processor_id, cpu_name_string()); if (c->options & MIPS_CPU_FPU) printk(KERN_INFO "FPU revision is: %08x\n", c->fpu_id); + if (cpu_has_msa) + pr_info("MSA revision is: %08x\n", c->msa_id); +} + +void cpu_set_cluster(struct cpuinfo_mips *cpuinfo, unsigned int cluster) +{ + /* Ensure the core number fits in the field */ + WARN_ON(cluster > (MIPS_GLOBALNUMBER_CLUSTER >> + MIPS_GLOBALNUMBER_CLUSTER_SHF)); + + cpuinfo->globalnumber &= ~MIPS_GLOBALNUMBER_CLUSTER; + cpuinfo->globalnumber |= cluster << MIPS_GLOBALNUMBER_CLUSTER_SHF; +} + +void cpu_set_core(struct cpuinfo_mips *cpuinfo, unsigned int core) +{ + /* Ensure the core number fits in the field */ + WARN_ON(core > (MIPS_GLOBALNUMBER_CORE >> MIPS_GLOBALNUMBER_CORE_SHF)); + + cpuinfo->globalnumber &= ~MIPS_GLOBALNUMBER_CORE; + cpuinfo->globalnumber |= core << MIPS_GLOBALNUMBER_CORE_SHF; +} + +void cpu_set_vpe_id(struct cpuinfo_mips *cpuinfo, unsigned int vpe) +{ + /* Ensure the VP(E) ID fits in the field */ + WARN_ON(vpe > (MIPS_GLOBALNUMBER_VP >> MIPS_GLOBALNUMBER_VP_SHF)); + + /* Ensure we're not using VP(E)s without support */ + WARN_ON(vpe && !IS_ENABLED(CONFIG_MIPS_MT_SMP) && + !IS_ENABLED(CONFIG_CPU_MIPSR6)); + + cpuinfo->globalnumber &= ~MIPS_GLOBALNUMBER_VP; + cpuinfo->globalnumber |= vpe << MIPS_GLOBALNUMBER_VP_SHF; +} + +void cpu_disable_mmid(void) +{ + int i; + unsigned long asid_mask; + unsigned int cpu = smp_processor_id(); + struct cpuinfo_mips *c = ¤t_cpu_data; + unsigned int config4 = read_c0_config4(); + unsigned int config5 = read_c0_config5(); + + /* Setup the initial ASID mask based on config4 */ + asid_mask = MIPS_ENTRYHI_ASID; + if (config4 & MIPS_CONF4_AE) + asid_mask |= MIPS_ENTRYHI_ASIDX; + set_cpu_asid_mask(c, asid_mask); + + /* Disable MMID in the C0 and update cpuinfo_mips accordingly */ + config5 &= ~(MIPS_CONF5_UFR | MIPS_CONF5_UFE); + config5 &= ~MIPS_CONF5_MI; + write_c0_config5(config5); + /* Ensure the write to config5 above takes effect */ + back_to_back_c0_hazard(); + c->options &= ~MIPS_CPU_MMID; + + /* Setup asid cache value cleared in per_cpu_trap_init() */ + cpu_data[cpu].asid_cache = asid_first_version(cpu); + + /* Reinit context for each CPU */ + for_each_possible_cpu(i) + set_cpu_context(i, &init_mm, 0); + + /* Ensure that now MMID will be seen as disable */ + mmid_disabled_quirk = true; + + pr_info("MMID support disabled due to hardware support issue\n"); } diff --git a/arch/mips/kernel/cpu-r3k-probe.c b/arch/mips/kernel/cpu-r3k-probe.c new file mode 100644 index 000000000000..0c826f729f75 --- /dev/null +++ b/arch/mips/kernel/cpu-r3k-probe.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Processor capabilities determination functions. + * + * Copyright (C) xxxx the Anonymous + * Copyright (C) 1994 - 2006 Ralf Baechle + * Copyright (C) 2003, 2004 Maciej W. Rozycki + * Copyright (C) 2001, 2004, 2011, 2012 MIPS Technologies, Inc. + */ +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/ptrace.h> +#include <linux/smp.h> +#include <linux/stddef.h> +#include <linux/export.h> + +#include <asm/bugs.h> +#include <asm/cpu.h> +#include <asm/cpu-features.h> +#include <asm/cpu-type.h> +#include <asm/fpu.h> +#include <asm/mipsregs.h> +#include <asm/elf.h> +#include <asm/traps.h> + +#include "fpu-probe.h" + +/* Hardware capabilities */ +unsigned int elf_hwcap __read_mostly; +EXPORT_SYMBOL_GPL(elf_hwcap); + +void __init check_bugs32(void) +{ + +} + +/* + * Probe whether cpu has config register by trying to play with + * alternate cache bit and see whether it matters. + * It's used by cpu_probe to distinguish between R3000A and R3081. + */ +static inline int cpu_has_confreg(void) +{ +#ifdef CONFIG_CPU_R3000 + unsigned long size1, size2; + unsigned long cfg = read_c0_conf(); + + size1 = r3k_cache_size(ST0_ISC); + write_c0_conf(cfg ^ R30XX_CONF_AC); + size2 = r3k_cache_size(ST0_ISC); + write_c0_conf(cfg); + return size1 != size2; +#else + return 0; +#endif +} + +static inline void set_elf_platform(int cpu, const char *plat) +{ + if (cpu == 0) + __elf_platform = plat; +} + +const char *__cpu_name[NR_CPUS]; +const char *__elf_platform; +const char *__elf_base_platform; + +void cpu_probe(void) +{ + struct cpuinfo_mips *c = ¤t_cpu_data; + unsigned int cpu = smp_processor_id(); + + /* + * Set a default elf platform, cpu probe may later + * overwrite it with a more precise value + */ + set_elf_platform(cpu, "mips"); + + c->processor_id = PRID_IMP_UNKNOWN; + c->fpu_id = FPIR_IMP_NONE; + c->cputype = CPU_UNKNOWN; + c->writecombine = _CACHE_UNCACHED; + + c->fpu_csr31 = FPU_CSR_RN; + c->fpu_msk31 = FPU_CSR_RSVD | FPU_CSR_ABS2008 | FPU_CSR_NAN2008 | + FPU_CSR_CONDX | FPU_CSR_FS; + + c->srsets = 1; + + c->processor_id = read_c0_prid(); + switch (c->processor_id & (PRID_COMP_MASK | PRID_IMP_MASK)) { + case PRID_COMP_LEGACY | PRID_IMP_R2000: + c->cputype = CPU_R2000; + __cpu_name[cpu] = "R2000"; + c->options = MIPS_CPU_TLB | MIPS_CPU_3K_CACHE | + MIPS_CPU_NOFPUEX; + if (__cpu_has_fpu()) + c->options |= MIPS_CPU_FPU; + c->tlbsize = 64; + break; + case PRID_COMP_LEGACY | PRID_IMP_R3000: + if ((c->processor_id & PRID_REV_MASK) == PRID_REV_R3000A) { + if (cpu_has_confreg()) { + c->cputype = CPU_R3081E; + __cpu_name[cpu] = "R3081"; + } else { + c->cputype = CPU_R3000A; + __cpu_name[cpu] = "R3000A"; + } + } else { + c->cputype = CPU_R3000; + __cpu_name[cpu] = "R3000"; + } + c->options = MIPS_CPU_TLB | MIPS_CPU_3K_CACHE | + MIPS_CPU_NOFPUEX; + if (__cpu_has_fpu()) + c->options |= MIPS_CPU_FPU; + c->tlbsize = 64; + break; + } + + BUG_ON(!__cpu_name[cpu]); + BUG_ON(c->cputype == CPU_UNKNOWN); + + /* + * Platform code can force the cpu type to optimize code + * generation. In that case be sure the cpu type is correctly + * manually setup otherwise it could trigger some nasty bugs. + */ + BUG_ON(current_cpu_type() != c->cputype); + + if (mips_fpu_disabled) + c->options &= ~MIPS_CPU_FPU; + + if (c->options & MIPS_CPU_FPU) + cpu_set_fpu_opts(c); + else + cpu_set_nofpu_opts(c); + + reserve_exception_space(0, 0x400); +} + +void cpu_report(void) +{ + struct cpuinfo_mips *c = ¤t_cpu_data; + + pr_info("CPU%d revision is: %08x (%s)\n", + smp_processor_id(), c->processor_id, cpu_name_string()); + if (c->options & MIPS_CPU_FPU) + pr_info("FPU revision is: %08x\n", c->fpu_id); +} diff --git a/arch/mips/kernel/crash.c b/arch/mips/kernel/crash.c index 93aa302948d7..81845ba04835 100644 --- a/arch/mips/kernel/crash.c +++ b/arch/mips/kernel/crash.c @@ -1,48 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/kernel.h> #include <linux/smp.h> #include <linux/reboot.h> #include <linux/kexec.h> -#include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/crash_dump.h> #include <linux/delay.h> -#include <linux/init.h> #include <linux/irq.h> #include <linux/types.h> #include <linux/sched.h> +#include <linux/sched/task_stack.h> /* This keeps a track of which one is crashing cpu. */ static int crashing_cpu = -1; static cpumask_t cpus_in_crash = CPU_MASK_NONE; #ifdef CONFIG_SMP -static void crash_shutdown_secondary(void *ignore) +static void crash_shutdown_secondary(void *passed_regs) { - struct pt_regs *regs; + struct pt_regs *regs = passed_regs; int cpu = smp_processor_id(); - regs = task_pt_regs(current); + /* + * If we are passed registers, use those. Otherwise get the + * regs from the last interrupt, which should be correct, as + * we are in an interrupt. But if the regs are not there, + * pull them from the top of the stack. They are probably + * wrong, but we need something to keep from crashing again. + */ + if (!regs) + regs = get_irq_regs(); + if (!regs) + regs = task_pt_regs(current); if (!cpu_online(cpu)) return; + /* We won't be sent IPIs any more. */ + set_cpu_online(cpu, false); + local_irq_disable(); - if (!cpu_isset(cpu, cpus_in_crash)) + if (!cpumask_test_cpu(cpu, &cpus_in_crash)) crash_save_cpu(regs, cpu); - cpu_set(cpu, cpus_in_crash); + cpumask_set_cpu(cpu, &cpus_in_crash); while (!atomic_read(&kexec_ready_to_reboot)) cpu_relax(); - relocated_kexec_smp_wait(NULL); + + kexec_reboot(); + /* NOTREACHED */ } static void crash_kexec_prepare_cpus(void) { + static int cpus_stopped; unsigned int msecs; + unsigned int ncpus; + + if (cpus_stopped) + return; - unsigned int ncpus = num_online_cpus() - 1;/* Excluding the panic cpu */ + ncpus = num_online_cpus() - 1;/* Excluding the panic cpu */ - dump_send_ipi(crash_shutdown_secondary); + smp_call_function(crash_shutdown_secondary, NULL, 0); smp_wmb(); /* @@ -51,10 +72,21 @@ static void crash_kexec_prepare_cpus(void) */ pr_emerg("Sending IPI to other cpus...\n"); msecs = 10000; - while ((cpus_weight(cpus_in_crash) < ncpus) && (--msecs > 0)) { + while ((cpumask_weight(&cpus_in_crash) < ncpus) && (--msecs > 0)) { cpu_relax(); mdelay(1); } + + cpus_stopped = 1; +} + +/* Override the weak function in kernel/panic.c */ +void crash_smp_send_stop(void) +{ + if (_crash_smp_send_stop) + _crash_smp_send_stop(); + + crash_kexec_prepare_cpus(); } #else /* !defined(CONFIG_SMP) */ @@ -67,5 +99,5 @@ void default_machine_crash_shutdown(struct pt_regs *regs) crashing_cpu = smp_processor_id(); crash_save_cpu(regs, crashing_cpu); crash_kexec_prepare_cpus(); - cpu_set(crashing_cpu, cpus_in_crash); + cpumask_set_cpu(crashing_cpu, &cpus_in_crash); } diff --git a/arch/mips/kernel/crash_dump.c b/arch/mips/kernel/crash_dump.c index f291cf99b03a..6e50f4902409 100644 --- a/arch/mips/kernel/crash_dump.c +++ b/arch/mips/kernel/crash_dump.c @@ -1,66 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/highmem.h> -#include <linux/bootmem.h> #include <linux/crash_dump.h> -#include <asm/uaccess.h> -#include <linux/slab.h> +#include <linux/uio.h> -static void *kdump_buf_page; - -/** - * copy_oldmem_page - copy one page from "oldmem" - * @pfn: page frame number to be copied - * @buf: target memory address for the copy; this can be in kernel address - * space or user address space (see @userbuf) - * @csize: number of bytes to copy - * @offset: offset in bytes into the page (based on pfn) to begin the copy - * @userbuf: if set, @buf is in user address space, use copy_to_user(), - * otherwise @buf is in kernel address space, use memcpy(). - * - * Copy a page from "oldmem". For this page, there is no pte mapped - * in the current kernel. - * - * Calling copy_to_user() in atomic context is not desirable. Hence first - * copying the data to a pre-allocated kernel page and then copying to user - * space in non-atomic context. - */ -ssize_t copy_oldmem_page(unsigned long pfn, char *buf, - size_t csize, unsigned long offset, int userbuf) +ssize_t copy_oldmem_page(struct iov_iter *iter, unsigned long pfn, + size_t csize, unsigned long offset) { void *vaddr; if (!csize) return 0; - vaddr = kmap_atomic_pfn(pfn); - - if (!userbuf) { - memcpy(buf, (vaddr + offset), csize); - kunmap_atomic(vaddr); - } else { - if (!kdump_buf_page) { - pr_warning("Kdump: Kdump buffer page not allocated\n"); - - return -EFAULT; - } - copy_page(kdump_buf_page, vaddr); - kunmap_atomic(vaddr); - if (copy_to_user(buf, (kdump_buf_page + offset), csize)) - return -EFAULT; - } + vaddr = kmap_local_pfn(pfn); + csize = copy_to_iter(vaddr + offset, csize, iter); + kunmap_local(vaddr); return csize; } - -static int __init kdump_buf_page_init(void) -{ - int ret = 0; - - kdump_buf_page = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!kdump_buf_page) { - pr_warning("Kdump: Failed to allocate kdump buffer page\n"); - ret = -ENOMEM; - } - - return ret; -} -arch_initcall(kdump_buf_page_init); diff --git a/arch/mips/kernel/csrc-bcm1480.c b/arch/mips/kernel/csrc-bcm1480.c index 468f3eba4132..6c18a138f897 100644 --- a/arch/mips/kernel/csrc-bcm1480.c +++ b/arch/mips/kernel/csrc-bcm1480.c @@ -1,21 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2000,2001,2004 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ #include <linux/clocksource.h> +#include <linux/sched_clock.h> #include <asm/addrspace.h> #include <asm/io.h> @@ -28,9 +16,9 @@ #include <asm/sibyte/sb1250.h> -static cycle_t bcm1480_hpt_read(struct clocksource *cs) +static u64 bcm1480_hpt_read(struct clocksource *cs) { - return (cycle_t) __raw_readq(IOADDR(A_SCD_ZBBUS_CYCLE_COUNT)); + return (u64) __raw_readq(IOADDR(A_SCD_ZBBUS_CYCLE_COUNT)); } struct clocksource bcm1480_clocksource = { @@ -41,6 +29,11 @@ struct clocksource bcm1480_clocksource = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; +static u64 notrace sb1480_read_sched_clock(void) +{ + return __raw_readq(IOADDR(A_SCD_ZBBUS_CYCLE_COUNT)); +} + void __init sb1480_clocksource_init(void) { struct clocksource *cs = &bcm1480_clocksource; @@ -50,4 +43,6 @@ void __init sb1480_clocksource_init(void) plldiv = G_BCM1480_SYS_PLL_DIV(__raw_readq(IOADDR(A_SCD_SYSTEM_CFG))); zbbus = ((plldiv >> 1) * 50000000) + ((plldiv & 1) * 25000000); clocksource_register_hz(cs, zbbus); + + sched_clock_register(sb1480_read_sched_clock, 64, zbbus); } diff --git a/arch/mips/kernel/csrc-gic.c b/arch/mips/kernel/csrc-gic.c deleted file mode 100644 index e02620901117..000000000000 --- a/arch/mips/kernel/csrc-gic.c +++ /dev/null @@ -1,40 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. - */ -#include <linux/init.h> -#include <linux/time.h> - -#include <asm/gic.h> - -static cycle_t gic_hpt_read(struct clocksource *cs) -{ - return gic_read_count(); -} - -static struct clocksource gic_clocksource = { - .name = "GIC", - .read = gic_hpt_read, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, -}; - -void __init gic_clocksource_init(unsigned int frequency) -{ - unsigned int config, bits; - - /* Calculate the clocksource mask. */ - GICREAD(GIC_REG(SHARED, GIC_SH_CONFIG), config); - bits = 32 + ((config & GIC_SH_CONFIG_COUNTBITS_MSK) >> - (GIC_SH_CONFIG_COUNTBITS_SHF - 2)); - - /* Set clocksource mask. */ - gic_clocksource.mask = CLOCKSOURCE_MASK(bits); - - /* Calculate a somewhat reasonable rating value. */ - gic_clocksource.rating = 200 + frequency / 10000000; - - clocksource_register_hz(&gic_clocksource, frequency); -} diff --git a/arch/mips/kernel/csrc-ioasic.c b/arch/mips/kernel/csrc-ioasic.c index 0654bff9b69c..bad740ad3218 100644 --- a/arch/mips/kernel/csrc-ioasic.c +++ b/arch/mips/kernel/csrc-ioasic.c @@ -1,23 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * DEC I/O ASIC's counter clocksource * * Copyright (C) 2008 Yoichi Yuasa <yuasa@linux-mips.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include <linux/clocksource.h> +#include <linux/sched_clock.h> #include <linux/init.h> #include <asm/ds1287.h> @@ -25,7 +13,7 @@ #include <asm/dec/ioasic.h> #include <asm/dec/ioasic_addrs.h> -static cycle_t dec_ioasic_hpt_read(struct clocksource *cs) +static u64 dec_ioasic_hpt_read(struct clocksource *cs) { return ioasic_read(IO_REG_FCTR); } @@ -37,13 +25,18 @@ static struct clocksource clocksource_dec = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; -void __init dec_ioasic_clocksource_init(void) +static u64 notrace dec_ioasic_read_sched_clock(void) +{ + return ioasic_read(IO_REG_FCTR); +} + +int __init dec_ioasic_clocksource_init(void) { unsigned int freq; u32 start, end; - int i = HZ / 10; - + int i = HZ / 8; + ds1287_timer_state(); while (!ds1287_timer_state()) ; @@ -55,9 +48,18 @@ void __init dec_ioasic_clocksource_init(void) end = dec_ioasic_hpt_read(&clocksource_dec); - freq = (end - start) * 10; + freq = (end - start) * 8; + + /* An early revision of the I/O ASIC didn't have the counter. */ + if (!freq) + return -ENXIO; + printk(KERN_INFO "I/O ASIC clock frequency %dHz\n", freq); clocksource_dec.rating = 200 + freq / 10000000; clocksource_register_hz(&clocksource_dec, freq); + + sched_clock_register(dec_ioasic_read_sched_clock, 32, freq); + + return 0; } diff --git a/arch/mips/kernel/csrc-powertv.c b/arch/mips/kernel/csrc-powertv.c deleted file mode 100644 index abd99ea911ae..000000000000 --- a/arch/mips/kernel/csrc-powertv.c +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (C) 2008 Scientific-Atlanta, Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - */ -/* - * The file comes from kernel/csrc-r4k.c - */ -#include <linux/clocksource.h> -#include <linux/init.h> - -#include <asm/time.h> /* Not included in linux/time.h */ - -#include <asm/mach-powertv/asic_regs.h> -#include "powertv-clock.h" - -/* MIPS PLL Register Definitions */ -#define PLL_GET_M(x) (((x) >> 8) & 0x000000FF) -#define PLL_GET_N(x) (((x) >> 16) & 0x000000FF) -#define PLL_GET_P(x) (((x) >> 24) & 0x00000007) - -/* - * returns: Clock frequency in kHz - */ -unsigned int __init mips_get_pll_freq(void) -{ - unsigned int pll_reg, m, n, p; - unsigned int fin = 54000; /* Base frequency in kHz */ - unsigned int fout; - - /* Read PLL register setting */ - pll_reg = asic_read(mips_pll_setup); - m = PLL_GET_M(pll_reg); - n = PLL_GET_N(pll_reg); - p = PLL_GET_P(pll_reg); - pr_info("MIPS PLL Register:0x%x M=%d N=%d P=%d\n", pll_reg, m, n, p); - - /* Calculate clock frequency = (2 * N * 54MHz) / (M * (2**P)) */ - fout = ((2 * n * fin) / (m * (0x01 << p))); - - pr_info("MIPS Clock Freq=%d kHz\n", fout); - - return fout; -} - -static cycle_t c0_hpt_read(struct clocksource *cs) -{ - return read_c0_count(); -} - -static struct clocksource clocksource_mips = { - .name = "powertv-counter", - .read = c0_hpt_read, - .mask = CLOCKSOURCE_MASK(32), - .flags = CLOCK_SOURCE_IS_CONTINUOUS, -}; - -static void __init powertv_c0_hpt_clocksource_init(void) -{ - unsigned int pll_freq = mips_get_pll_freq(); - - pr_info("CPU frequency %d.%02d MHz\n", pll_freq / 1000, - (pll_freq % 1000) * 100 / 1000); - - mips_hpt_frequency = pll_freq / 2 * 1000; - - clocksource_mips.rating = 200 + mips_hpt_frequency / 10000000; - - clocksource_register_hz(&clocksource_mips, mips_hpt_frequency); -} - -/** - * struct tim_c - free running counter - * @hi: High 16 bits of the counter - * @lo: Low 32 bits of the counter - * - * Lays out the structure of the free running counter in memory. This counter - * increments at a rate of 27 MHz/8 on all platforms. - */ -struct tim_c { - unsigned int hi; - unsigned int lo; -}; - -static struct tim_c *tim_c; - -static cycle_t tim_c_read(struct clocksource *cs) -{ - unsigned int hi; - unsigned int next_hi; - unsigned int lo; - - hi = readl(&tim_c->hi); - - for (;;) { - lo = readl(&tim_c->lo); - next_hi = readl(&tim_c->hi); - if (next_hi == hi) - break; - hi = next_hi; - } - -pr_crit("%s: read %llx\n", __func__, ((u64) hi << 32) | lo); - return ((u64) hi << 32) | lo; -} - -#define TIM_C_SIZE 48 /* # bits in the timer */ - -static struct clocksource clocksource_tim_c = { - .name = "powertv-tim_c", - .read = tim_c_read, - .mask = CLOCKSOURCE_MASK(TIM_C_SIZE), - .flags = CLOCK_SOURCE_IS_CONTINUOUS, -}; - -/** - * powertv_tim_c_clocksource_init - set up a clock source for the TIM_C clock - * - * We know that TIM_C counts at 27 MHz/8, so each cycle corresponds to - * 1 / (27,000,000/8) seconds. - */ -static void __init powertv_tim_c_clocksource_init(void) -{ - const unsigned long counts_per_second = 27000000 / 8; - - clocksource_tim_c.rating = 200; - - clocksource_register_hz(&clocksource_tim_c, counts_per_second); - tim_c = (struct tim_c *) asic_reg_addr(tim_ch); -} - -/** - powertv_clocksource_init - initialize all clocksources - */ -void __init powertv_clocksource_init(void) -{ - powertv_c0_hpt_clocksource_init(); - powertv_tim_c_clocksource_init(); -} diff --git a/arch/mips/kernel/csrc-r4k.c b/arch/mips/kernel/csrc-r4k.c index decd1fa38d55..59eca397f297 100644 --- a/arch/mips/kernel/csrc-r4k.c +++ b/arch/mips/kernel/csrc-r4k.c @@ -6,11 +6,13 @@ * Copyright (C) 2007 by Ralf Baechle */ #include <linux/clocksource.h> +#include <linux/cpufreq.h> #include <linux/init.h> +#include <linux/sched_clock.h> #include <asm/time.h> -static cycle_t c0_hpt_read(struct clocksource *cs) +static u64 c0_hpt_read(struct clocksource *cs) { return read_c0_count(); } @@ -22,15 +24,119 @@ static struct clocksource clocksource_mips = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; +static u64 __maybe_unused notrace r4k_read_sched_clock(void) +{ + return read_c0_count(); +} + +static inline unsigned int rdhwr_count(void) +{ + unsigned int count; + + __asm__ __volatile__( + " .set push\n" + " .set mips32r2\n" + " rdhwr %0, $2\n" + " .set pop\n" + : "=r" (count)); + + return count; +} + +static bool rdhwr_count_usable(void) +{ + unsigned int prev, curr, i; + + /* + * Older QEMUs have a broken implementation of RDHWR for the CP0 count + * which always returns a constant value. Try to identify this and don't + * use it in the VDSO if it is broken. This workaround can be removed + * once the fix has been in QEMU stable for a reasonable amount of time. + */ + for (i = 0, prev = rdhwr_count(); i < 100; i++) { + curr = rdhwr_count(); + + if (curr != prev) + return true; + + prev = curr; + } + + pr_warn("Not using R4K clocksource in VDSO due to broken RDHWR\n"); + return false; +} + +static inline __init bool count_can_be_sched_clock(void) +{ + if (IS_ENABLED(CONFIG_CPU_FREQ)) + return false; + + if (num_possible_cpus() > 1 && + !IS_ENABLED(CONFIG_HAVE_UNSTABLE_SCHED_CLOCK)) + return false; + + return true; +} + +#ifdef CONFIG_CPU_FREQ + +static bool __read_mostly r4k_clock_unstable; + +static void r4k_clocksource_unstable(char *reason) +{ + if (r4k_clock_unstable) + return; + + r4k_clock_unstable = true; + + pr_info("R4K timer is unstable due to %s\n", reason); + + clocksource_mark_unstable(&clocksource_mips); +} + +static int r4k_cpufreq_callback(struct notifier_block *nb, + unsigned long val, void *data) +{ + if (val == CPUFREQ_POSTCHANGE) + r4k_clocksource_unstable("CPU frequency change"); + + return 0; +} + +static struct notifier_block r4k_cpufreq_notifier = { + .notifier_call = r4k_cpufreq_callback, +}; + +static int __init r4k_register_cpufreq_notifier(void) +{ + return cpufreq_register_notifier(&r4k_cpufreq_notifier, + CPUFREQ_TRANSITION_NOTIFIER); + +} +core_initcall(r4k_register_cpufreq_notifier); + +#endif /* !CONFIG_CPU_FREQ */ + int __init init_r4k_clocksource(void) { if (!cpu_has_counter || !mips_hpt_frequency) return -ENXIO; /* Calculate a somewhat reasonable rating value */ - clocksource_mips.rating = 200 + mips_hpt_frequency / 10000000; + clocksource_mips.rating = 200; + clocksource_mips.rating += clamp(mips_hpt_frequency / 10000000, 0, 99); + + /* + * R2 onwards makes the count accessible to user mode so it can be used + * by the VDSO (HWREna is configured by configure_hwrena()). + */ + if (cpu_has_mips_r2_r6 && rdhwr_count_usable()) + clocksource_mips.vdso_clock_mode = VDSO_CLOCKMODE_R4K; clocksource_register_hz(&clocksource_mips, mips_hpt_frequency); + if (count_can_be_sched_clock()) + sched_clock_register(r4k_read_sched_clock, 32, mips_hpt_frequency); + return 0; } diff --git a/arch/mips/kernel/csrc-sb1250.c b/arch/mips/kernel/csrc-sb1250.c index 6ecb77d82063..fa2fa3e104ac 100644 --- a/arch/mips/kernel/csrc-sb1250.c +++ b/arch/mips/kernel/csrc-sb1250.c @@ -1,21 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2000, 2001 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ #include <linux/clocksource.h> +#include <linux/sched_clock.h> #include <asm/addrspace.h> #include <asm/io.h> @@ -33,15 +21,22 @@ * The HPT is free running from SB1250_HPT_VALUE down to 0 then starts over * again. */ -static cycle_t sb1250_hpt_read(struct clocksource *cs) +static inline u64 sb1250_hpt_get_cycles(void) { unsigned int count; + void __iomem *addr; - count = G_SCD_TIMER_CNT(__raw_readq(IOADDR(A_SCD_TIMER_REGISTER(SB1250_HPT_NUM, R_SCD_TIMER_CNT)))); + addr = IOADDR(A_SCD_TIMER_REGISTER(SB1250_HPT_NUM, R_SCD_TIMER_CNT)); + count = G_SCD_TIMER_CNT(__raw_readq(addr)); return SB1250_HPT_VALUE - count; } +static u64 sb1250_hpt_read(struct clocksource *cs) +{ + return sb1250_hpt_get_cycles(); +} + struct clocksource bcm1250_clocksource = { .name = "bcm1250-counter-3", .rating = 200, @@ -50,6 +45,11 @@ struct clocksource bcm1250_clocksource = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; +static u64 notrace sb1250_read_sched_clock(void) +{ + return sb1250_hpt_get_cycles(); +} + void __init sb1250_clocksource_init(void) { struct clocksource *cs = &bcm1250_clocksource; @@ -66,4 +66,6 @@ void __init sb1250_clocksource_init(void) R_SCD_TIMER_CFG))); clocksource_register_hz(cs, V_SCD_TIMER_FREQ); + + sched_clock_register(sb1250_read_sched_clock, 23, V_SCD_TIMER_FREQ); } diff --git a/arch/mips/kernel/early_printk.c b/arch/mips/kernel/early_printk.c index 505cb77d1280..4a1647ddfbd9 100644 --- a/arch/mips/kernel/early_printk.c +++ b/arch/mips/kernel/early_printk.c @@ -14,8 +14,6 @@ #include <asm/setup.h> -extern void prom_putchar(char); - static void early_console_write(struct console *con, const char *s, unsigned n) { while (n-- && *s) { diff --git a/arch/mips/kernel/early_printk_8250.c b/arch/mips/kernel/early_printk_8250.c new file mode 100644 index 000000000000..567c6ec0cfae --- /dev/null +++ b/arch/mips/kernel/early_printk_8250.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * 8250/16550-type serial ports prom_putchar() + * + * Copyright (C) 2010 Yoichi Yuasa <yuasa@linux-mips.org> + */ +#include <linux/io.h> +#include <linux/serial_core.h> +#include <linux/serial_reg.h> +#include <asm/setup.h> + +static void __iomem *serial8250_base; +static unsigned int serial8250_reg_shift; +static unsigned int serial8250_tx_timeout; + +void setup_8250_early_printk_port(unsigned long base, unsigned int reg_shift, + unsigned int timeout) +{ + serial8250_base = (void __iomem *)base; + serial8250_reg_shift = reg_shift; + serial8250_tx_timeout = timeout; +} + +static inline u8 serial_in(int offset) +{ + return readb(serial8250_base + (offset << serial8250_reg_shift)); +} + +static inline void serial_out(int offset, char value) +{ + writeb(value, serial8250_base + (offset << serial8250_reg_shift)); +} + +void prom_putchar(char c) +{ + unsigned int timeout; + int status, bits; + + if (!serial8250_base) + return; + + timeout = serial8250_tx_timeout; + bits = UART_LSR_TEMT | UART_LSR_THRE; + + do { + status = serial_in(UART_LSR); + + if (--timeout == 0) + break; + } while ((status & bits) != bits); + + if (timeout) + serial_out(UART_TX, c); +} diff --git a/arch/mips/kernel/elf.c b/arch/mips/kernel/elf.c new file mode 100644 index 000000000000..f0e7fe85a42a --- /dev/null +++ b/arch/mips/kernel/elf.c @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2014 Imagination Technologies + * Author: Paul Burton <paul.burton@mips.com> + */ + +#include <linux/binfmts.h> +#include <linux/elf.h> +#include <linux/export.h> +#include <linux/sched.h> + +#include <asm/cpu-features.h> +#include <asm/cpu-info.h> +#include <asm/fpu.h> + +#ifdef CONFIG_MIPS_FP_SUPPORT + +/* Whether to accept legacy-NaN and 2008-NaN user binaries. */ +bool mips_use_nan_legacy; +bool mips_use_nan_2008; + +/* FPU modes */ +enum { + FP_FRE, + FP_FR0, + FP_FR1, +}; + +/** + * struct mode_req - ABI FPU mode requirements + * @single: The program being loaded needs an FPU but it will only issue + * single precision instructions meaning that it can execute in + * either FR0 or FR1. + * @soft: The soft(-float) requirement means that the program being + * loaded needs has no FPU dependency at all (i.e. it has no + * FPU instructions). + * @fr1: The program being loaded depends on FPU being in FR=1 mode. + * @frdefault: The program being loaded depends on the default FPU mode. + * That is FR0 for O32 and FR1 for N32/N64. + * @fre: The program being loaded depends on FPU with FRE=1. This mode is + * a bridge which uses FR=1 whilst still being able to maintain + * full compatibility with pre-existing code using the O32 FP32 + * ABI. + * + * More information about the FP ABIs can be found here: + * + * https://dmz-portal.mips.com/wiki/MIPS_O32_ABI_-_FR0_and_FR1_Interlinking#10.4.1._Basic_mode_set-up + * + */ + +struct mode_req { + bool single; + bool soft; + bool fr1; + bool frdefault; + bool fre; +}; + +static const struct mode_req fpu_reqs[] = { + [MIPS_ABI_FP_ANY] = { true, true, true, true, true }, + [MIPS_ABI_FP_DOUBLE] = { false, false, false, true, true }, + [MIPS_ABI_FP_SINGLE] = { true, false, false, false, false }, + [MIPS_ABI_FP_SOFT] = { false, true, false, false, false }, + [MIPS_ABI_FP_OLD_64] = { false, false, false, false, false }, + [MIPS_ABI_FP_XX] = { false, false, true, true, true }, + [MIPS_ABI_FP_64] = { false, false, true, false, false }, + [MIPS_ABI_FP_64A] = { false, false, true, false, true } +}; + +/* + * Mode requirements when .MIPS.abiflags is not present in the ELF. + * Not present means that everything is acceptable except FR1. + */ +static struct mode_req none_req = { true, true, false, true, true }; + +int arch_elf_pt_proc(void *_ehdr, void *_phdr, struct file *elf, + bool is_interp, struct arch_elf_state *state) +{ + union { + struct elf32_hdr e32; + struct elf64_hdr e64; + } *ehdr = _ehdr; + struct elf32_phdr *phdr32 = _phdr; + struct elf64_phdr *phdr64 = _phdr; + struct mips_elf_abiflags_v0 abiflags; + bool elf32; + u32 flags; + int ret; + loff_t pos; + + elf32 = ehdr->e32.e_ident[EI_CLASS] == ELFCLASS32; + flags = elf32 ? ehdr->e32.e_flags : ehdr->e64.e_flags; + + /* Let's see if this is an O32 ELF */ + if (elf32) { + if (flags & EF_MIPS_FP64) { + /* + * Set MIPS_ABI_FP_OLD_64 for EF_MIPS_FP64. We will override it + * later if needed + */ + if (is_interp) + state->interp_fp_abi = MIPS_ABI_FP_OLD_64; + else + state->fp_abi = MIPS_ABI_FP_OLD_64; + } + if (phdr32->p_type != PT_MIPS_ABIFLAGS) + return 0; + + if (phdr32->p_filesz < sizeof(abiflags)) + return -EINVAL; + pos = phdr32->p_offset; + } else { + if (phdr64->p_type != PT_MIPS_ABIFLAGS) + return 0; + if (phdr64->p_filesz < sizeof(abiflags)) + return -EINVAL; + pos = phdr64->p_offset; + } + + ret = kernel_read(elf, &abiflags, sizeof(abiflags), &pos); + if (ret < 0) + return ret; + if (ret != sizeof(abiflags)) + return -EIO; + + /* Record the required FP ABIs for use by mips_check_elf */ + if (is_interp) + state->interp_fp_abi = abiflags.fp_abi; + else + state->fp_abi = abiflags.fp_abi; + + return 0; +} + +int arch_check_elf(void *_ehdr, bool has_interpreter, void *_interp_ehdr, + struct arch_elf_state *state) +{ + union { + struct elf32_hdr e32; + struct elf64_hdr e64; + } *ehdr = _ehdr; + union { + struct elf32_hdr e32; + struct elf64_hdr e64; + } *iehdr = _interp_ehdr; + struct mode_req prog_req, interp_req; + int fp_abi, interp_fp_abi, abi0, abi1, max_abi; + bool elf32; + u32 flags; + + elf32 = ehdr->e32.e_ident[EI_CLASS] == ELFCLASS32; + flags = elf32 ? ehdr->e32.e_flags : ehdr->e64.e_flags; + + /* + * Determine the NaN personality, reject the binary if not allowed. + * Also ensure that any interpreter matches the executable. + */ + if (flags & EF_MIPS_NAN2008) { + if (mips_use_nan_2008) + state->nan_2008 = 1; + else + return -ENOEXEC; + } else { + if (mips_use_nan_legacy) + state->nan_2008 = 0; + else + return -ENOEXEC; + } + if (has_interpreter) { + bool ielf32; + u32 iflags; + + ielf32 = iehdr->e32.e_ident[EI_CLASS] == ELFCLASS32; + iflags = ielf32 ? iehdr->e32.e_flags : iehdr->e64.e_flags; + + if ((flags ^ iflags) & EF_MIPS_NAN2008) + return -ELIBBAD; + } + + if (!IS_ENABLED(CONFIG_MIPS_O32_FP64_SUPPORT)) + return 0; + + fp_abi = state->fp_abi; + + if (has_interpreter) { + interp_fp_abi = state->interp_fp_abi; + + abi0 = min(fp_abi, interp_fp_abi); + abi1 = max(fp_abi, interp_fp_abi); + } else { + abi0 = abi1 = fp_abi; + } + + if (elf32 && !(flags & EF_MIPS_ABI2)) { + /* Default to a mode capable of running code expecting FR=0 */ + state->overall_fp_mode = cpu_has_mips_r6 ? FP_FRE : FP_FR0; + + /* Allow all ABIs we know about */ + max_abi = MIPS_ABI_FP_64A; + } else { + /* MIPS64 code always uses FR=1, thus the default is easy */ + state->overall_fp_mode = FP_FR1; + + /* Disallow access to the various FPXX & FP64 ABIs */ + max_abi = MIPS_ABI_FP_SOFT; + } + + if ((abi0 > max_abi && abi0 != MIPS_ABI_FP_UNKNOWN) || + (abi1 > max_abi && abi1 != MIPS_ABI_FP_UNKNOWN)) + return -ELIBBAD; + + /* It's time to determine the FPU mode requirements */ + prog_req = (abi0 == MIPS_ABI_FP_UNKNOWN) ? none_req : fpu_reqs[abi0]; + interp_req = (abi1 == MIPS_ABI_FP_UNKNOWN) ? none_req : fpu_reqs[abi1]; + + /* + * Check whether the program's and interp's ABIs have a matching FPU + * mode requirement. + */ + prog_req.single = interp_req.single && prog_req.single; + prog_req.soft = interp_req.soft && prog_req.soft; + prog_req.fr1 = interp_req.fr1 && prog_req.fr1; + prog_req.frdefault = interp_req.frdefault && prog_req.frdefault; + prog_req.fre = interp_req.fre && prog_req.fre; + + /* + * Determine the desired FPU mode + * + * Decision making: + * + * - We want FR_FRE if FRE=1 and both FR=1 and FR=0 are false. This + * means that we have a combination of program and interpreter + * that inherently require the hybrid FP mode. + * - If FR1 and FRDEFAULT is true, that means we hit the any-abi or + * fpxx case. This is because, in any-ABI (or no-ABI) we have no FPU + * instructions so we don't care about the mode. We will simply use + * the one preferred by the hardware. In fpxx case, that ABI can + * handle both FR=1 and FR=0, so, again, we simply choose the one + * preferred by the hardware. Next, if we only use single-precision + * FPU instructions, and the default ABI FPU mode is not good + * (ie single + any ABI combination), we set again the FPU mode to the + * one is preferred by the hardware. Next, if we know that the code + * will only use single-precision instructions, shown by single being + * true but frdefault being false, then we again set the FPU mode to + * the one that is preferred by the hardware. + * - We want FP_FR1 if that's the only matching mode and the default one + * is not good. + * - Return with -ELIBADD if we can't find a matching FPU mode. + */ + if (prog_req.fre && !prog_req.frdefault && !prog_req.fr1) + state->overall_fp_mode = FP_FRE; + else if ((prog_req.fr1 && prog_req.frdefault) || + (prog_req.single && !prog_req.frdefault)) + /* Make sure 64-bit MIPS III/IV/64R1 will not pick FR1 */ + state->overall_fp_mode = ((raw_current_cpu_data.fpu_id & MIPS_FPIR_F64) && + cpu_has_mips_r2_r6) ? + FP_FR1 : FP_FR0; + else if (prog_req.fr1) + state->overall_fp_mode = FP_FR1; + else if (!prog_req.fre && !prog_req.frdefault && + !prog_req.fr1 && !prog_req.single && !prog_req.soft) + return -ELIBBAD; + + return 0; +} + +static inline void set_thread_fp_mode(int hybrid, int regs32) +{ + if (hybrid) + set_thread_flag(TIF_HYBRID_FPREGS); + else + clear_thread_flag(TIF_HYBRID_FPREGS); + if (regs32) + set_thread_flag(TIF_32BIT_FPREGS); + else + clear_thread_flag(TIF_32BIT_FPREGS); +} + +void mips_set_personality_fp(struct arch_elf_state *state) +{ + /* + * This function is only ever called for O32 ELFs so we should + * not be worried about N32/N64 binaries. + */ + + if (!IS_ENABLED(CONFIG_MIPS_O32_FP64_SUPPORT)) + return; + + switch (state->overall_fp_mode) { + case FP_FRE: + set_thread_fp_mode(1, 0); + break; + case FP_FR0: + set_thread_fp_mode(0, 1); + break; + case FP_FR1: + set_thread_fp_mode(0, 0); + break; + default: + BUG(); + } +} + +/* + * Select the IEEE 754 NaN encoding and ABS.fmt/NEG.fmt execution mode + * in FCSR according to the ELF NaN personality. + */ +void mips_set_personality_nan(struct arch_elf_state *state) +{ + struct cpuinfo_mips *c = &boot_cpu_data; + struct task_struct *t = current; + + /* Do this early so t->thread.fpu.fcr31 won't be clobbered in case + * we are preempted before the lose_fpu(0) in start_thread. + */ + lose_fpu(0); + + t->thread.fpu.fcr31 = c->fpu_csr31; + switch (state->nan_2008) { + case 0: + if (!(c->fpu_msk31 & FPU_CSR_NAN2008)) + t->thread.fpu.fcr31 &= ~FPU_CSR_NAN2008; + if (!(c->fpu_msk31 & FPU_CSR_ABS2008)) + t->thread.fpu.fcr31 &= ~FPU_CSR_ABS2008; + break; + case 1: + if (!(c->fpu_msk31 & FPU_CSR_NAN2008)) + t->thread.fpu.fcr31 |= FPU_CSR_NAN2008; + if (!(c->fpu_msk31 & FPU_CSR_ABS2008)) + t->thread.fpu.fcr31 |= FPU_CSR_ABS2008; + break; + default: + BUG(); + } +} + +#endif /* CONFIG_MIPS_FP_SUPPORT */ + +int mips_elf_read_implies_exec(void *elf_ex, int exstack) +{ + /* + * Set READ_IMPLIES_EXEC only on non-NX systems that + * do not request a specific state via PT_GNU_STACK. + */ + return (!cpu_has_rixi && exstack == EXSTACK_DEFAULT); +} +EXPORT_SYMBOL(mips_elf_read_implies_exec); diff --git a/arch/mips/kernel/entry.S b/arch/mips/kernel/entry.S index e5786858cdb6..891393626dc6 100644 --- a/arch/mips/kernel/entry.S +++ b/arch/mips/kernel/entry.S @@ -10,17 +10,15 @@ #include <asm/asm.h> #include <asm/asmmacro.h> +#include <asm/compiler.h> +#include <asm/irqflags.h> #include <asm/regdef.h> #include <asm/mipsregs.h> #include <asm/stackframe.h> #include <asm/isadep.h> #include <asm/thread_info.h> -#include <asm/war.h> -#ifdef CONFIG_MIPS_MT_SMTC -#include <asm/mipsmtregs.h> -#endif -#ifndef CONFIG_PREEMPT +#ifndef CONFIG_PREEMPTION #define resume_kernel restore_all #else #define __ret_from_irq ret_from_exception @@ -28,7 +26,7 @@ .text .align 5 -#ifndef CONFIG_PREEMPT +#ifndef CONFIG_PREEMPTION FEXPORT(ret_from_exception) local_irq_disable # preempt stop b __ret_from_irq @@ -54,20 +52,19 @@ resume_userspace: bnez t0, work_pending j restore_all -#ifdef CONFIG_PREEMPT +#ifdef CONFIG_PREEMPTION resume_kernel: local_irq_disable lw t0, TI_PRE_COUNT($28) bnez t0, restore_all -need_resched: LONG_L t0, TI_FLAGS($28) andi t1, t0, _TIF_NEED_RESCHED beqz t1, restore_all LONG_L t0, PT_STATUS(sp) # Interrupts off? andi t0, 1 beqz t0, restore_all - jal preempt_schedule_irq - b need_resched + PTR_LA ra, restore_all + j preempt_schedule_irq #endif FEXPORT(ret_from_kernel_thread) @@ -80,6 +77,10 @@ FEXPORT(ret_from_fork) jal schedule_tail # a0 = struct task_struct *prev FEXPORT(syscall_exit) +#ifdef CONFIG_DEBUG_RSEQ + move a0, sp + jal rseq_syscall +#endif local_irq_disable # make sure need_resched and # signals dont change between # sampling and return @@ -89,41 +90,6 @@ FEXPORT(syscall_exit) bnez t0, syscall_exit_work restore_all: # restore full frame -#ifdef CONFIG_MIPS_MT_SMTC -#ifdef CONFIG_MIPS_MT_SMTC_IM_BACKSTOP -/* Re-arm any temporarily masked interrupts not explicitly "acked" */ - mfc0 v0, CP0_TCSTATUS - ori v1, v0, TCSTATUS_IXMT - mtc0 v1, CP0_TCSTATUS - andi v0, TCSTATUS_IXMT - _ehb - mfc0 t0, CP0_TCCONTEXT - DMT 9 # dmt t1 - jal mips_ihb - mfc0 t2, CP0_STATUS - andi t3, t0, 0xff00 - or t2, t2, t3 - mtc0 t2, CP0_STATUS - _ehb - andi t1, t1, VPECONTROL_TE - beqz t1, 1f - EMT -1: - mfc0 v1, CP0_TCSTATUS - /* We set IXMT above, XOR should clear it here */ - xori v1, v1, TCSTATUS_IXMT - or v1, v0, v1 - mtc0 v1, CP0_TCSTATUS - _ehb - xor t0, t0, t3 - mtc0 t0, CP0_TCCONTEXT -#endif /* CONFIG_MIPS_MT_SMTC_IM_BACKSTOP */ -/* Detect and execute deferred IPI "interrupts" */ - LONG_L s0, TI_REGS($28) - LONG_S sp, TI_REGS($28) - jal deferred_smtc_ipi - LONG_S s0, TI_REGS($28) -#endif /* CONFIG_MIPS_MT_SMTC */ .set noat RESTORE_TEMP RESTORE_AT @@ -134,7 +100,7 @@ restore_partial: # restore partial frame SAVE_AT SAVE_TEMP LONG_L v0, PT_STATUS(sp) -#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX) +#if defined(CONFIG_CPU_R3000) and v0, ST0_IEP #else and v0, ST0_IE @@ -156,6 +122,7 @@ work_pending: andi t0, a2, _TIF_NEED_RESCHED # a2 is preloaded with TI_FLAGS beqz t0, work_notifysig work_resched: + TRACE_IRQS_OFF jal schedule local_irq_disable # make sure need_resched and @@ -176,6 +143,10 @@ work_notifysig: # deal with pending signals and j resume_userspace_check FEXPORT(syscall_exit_partial) +#ifdef CONFIG_DEBUG_RSEQ + move a0, sp + jal rseq_syscall +#endif local_irq_disable # make sure need_resched doesn't # change between and return LONG_L a2, TI_FLAGS($28) # current->work @@ -192,11 +163,13 @@ syscall_exit_work: beqz t0, work_pending # trace bit set? local_irq_enable # could let syscall_trace_leave() # call schedule() instead + TRACE_IRQS_ON move a0, sp jal syscall_trace_leave b resume_userspace -#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_MIPS_MT) +#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \ + defined(CONFIG_CPU_MIPSR6) || defined(CONFIG_MIPS_MT) /* * MIPS32R2 Instruction Hazard Barrier - must be called @@ -204,9 +177,9 @@ syscall_exit_work: * For C code use the inline version named instruction_hazard(). */ LEAF(mips_ihb) - .set mips32r2 + .set MIPS_ISA_LEVEL_RAW jr.hb ra nop END(mips_ihb) -#endif /* CONFIG_CPU_MIPSR2 or CONFIG_MIPS_MT */ +#endif /* CONFIG_CPU_MIPSR2 - CONFIG_CPU_MIPSR6 or CONFIG_MIPS_MT */ diff --git a/arch/mips/kernel/fpu-probe.c b/arch/mips/kernel/fpu-probe.c new file mode 100644 index 000000000000..6bf3f19b1c33 --- /dev/null +++ b/arch/mips/kernel/fpu-probe.c @@ -0,0 +1,328 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Processor capabilities determination functions. + * + * Copyright (C) xxxx the Anonymous + * Copyright (C) 1994 - 2006 Ralf Baechle + * Copyright (C) 2003, 2004 Maciej W. Rozycki + * Copyright (C) 2001, 2004, 2011, 2012 MIPS Technologies, Inc. + */ + +#include <linux/init.h> +#include <linux/kernel.h> + +#include <asm/bugs.h> +#include <asm/cpu.h> +#include <asm/cpu-features.h> +#include <asm/cpu-type.h> +#include <asm/elf.h> +#include <asm/fpu.h> +#include <asm/mipsregs.h> + +#include "fpu-probe.h" + +/* + * Get the FPU Implementation/Revision. + */ +static inline unsigned long cpu_get_fpu_id(void) +{ + unsigned long tmp, fpu_id; + + tmp = read_c0_status(); + __enable_fpu(FPU_AS_IS); + fpu_id = read_32bit_cp1_register(CP1_REVISION); + write_c0_status(tmp); + return fpu_id; +} + +/* + * Check if the CPU has an external FPU. + */ +int __cpu_has_fpu(void) +{ + return (cpu_get_fpu_id() & FPIR_IMP_MASK) != FPIR_IMP_NONE; +} + +/* + * Determine the FCSR mask for FPU hardware. + */ +static inline void cpu_set_fpu_fcsr_mask(struct cpuinfo_mips *c) +{ + unsigned long sr, mask, fcsr, fcsr0, fcsr1; + + fcsr = c->fpu_csr31; + mask = FPU_CSR_ALL_X | FPU_CSR_ALL_E | FPU_CSR_ALL_S | FPU_CSR_RM; + + sr = read_c0_status(); + __enable_fpu(FPU_AS_IS); + + fcsr0 = fcsr & mask; + write_32bit_cp1_register(CP1_STATUS, fcsr0); + fcsr0 = read_32bit_cp1_register(CP1_STATUS); + + fcsr1 = fcsr | ~mask; + write_32bit_cp1_register(CP1_STATUS, fcsr1); + fcsr1 = read_32bit_cp1_register(CP1_STATUS); + + write_32bit_cp1_register(CP1_STATUS, fcsr); + + write_c0_status(sr); + + c->fpu_msk31 = ~(fcsr0 ^ fcsr1) & ~mask; +} + +/* + * Determine the IEEE 754 NaN encodings and ABS.fmt/NEG.fmt execution modes + * supported by FPU hardware. + */ +static void cpu_set_fpu_2008(struct cpuinfo_mips *c) +{ + if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 | + MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 | + MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 | + MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) { + unsigned long sr, fir, fcsr, fcsr0, fcsr1; + + sr = read_c0_status(); + __enable_fpu(FPU_AS_IS); + + fir = read_32bit_cp1_register(CP1_REVISION); + if (fir & MIPS_FPIR_HAS2008) { + fcsr = read_32bit_cp1_register(CP1_STATUS); + + /* + * MAC2008 toolchain never landed in real world, so + * we're only testing whether it can be disabled and + * don't try to enabled it. + */ + fcsr0 = fcsr & ~(FPU_CSR_ABS2008 | FPU_CSR_NAN2008 | + FPU_CSR_MAC2008); + write_32bit_cp1_register(CP1_STATUS, fcsr0); + fcsr0 = read_32bit_cp1_register(CP1_STATUS); + + fcsr1 = fcsr | FPU_CSR_ABS2008 | FPU_CSR_NAN2008; + write_32bit_cp1_register(CP1_STATUS, fcsr1); + fcsr1 = read_32bit_cp1_register(CP1_STATUS); + + write_32bit_cp1_register(CP1_STATUS, fcsr); + + if (c->isa_level & (MIPS_CPU_ISA_M32R2 | + MIPS_CPU_ISA_M64R2)) { + /* + * The bit for MAC2008 might be reused by R6 + * in future, so we only test for R2-R5. + */ + if (fcsr0 & FPU_CSR_MAC2008) + c->options |= MIPS_CPU_MAC_2008_ONLY; + } + + if (!(fcsr0 & FPU_CSR_NAN2008)) + c->options |= MIPS_CPU_NAN_LEGACY; + if (fcsr1 & FPU_CSR_NAN2008) + c->options |= MIPS_CPU_NAN_2008; + + if ((fcsr0 ^ fcsr1) & FPU_CSR_ABS2008) + c->fpu_msk31 &= ~FPU_CSR_ABS2008; + else + c->fpu_csr31 |= fcsr & FPU_CSR_ABS2008; + + if ((fcsr0 ^ fcsr1) & FPU_CSR_NAN2008) + c->fpu_msk31 &= ~FPU_CSR_NAN2008; + else + c->fpu_csr31 |= fcsr & FPU_CSR_NAN2008; + } else { + c->options |= MIPS_CPU_NAN_LEGACY; + } + + write_c0_status(sr); + } else { + c->options |= MIPS_CPU_NAN_LEGACY; + } +} + +/* + * IEEE 754 conformance mode to use. Affects the NaN encoding and the + * ABS.fmt/NEG.fmt execution mode. + */ +static enum { STRICT, EMULATED, LEGACY, STD2008, RELAXED } ieee754 = STRICT; + +/* + * Set the IEEE 754 NaN encodings and the ABS.fmt/NEG.fmt execution modes + * to support by the FPU emulator according to the IEEE 754 conformance + * mode selected. Note that "relaxed" straps the emulator so that it + * allows 2008-NaN binaries even for legacy processors. + */ +static void cpu_set_nofpu_2008(struct cpuinfo_mips *c) +{ + c->options &= ~(MIPS_CPU_NAN_2008 | MIPS_CPU_NAN_LEGACY); + c->fpu_csr31 &= ~(FPU_CSR_ABS2008 | FPU_CSR_NAN2008); + c->fpu_msk31 &= ~(FPU_CSR_ABS2008 | FPU_CSR_NAN2008); + + switch (ieee754) { + case STRICT: + case EMULATED: + if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 | + MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 | + MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 | + MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) { + c->options |= MIPS_CPU_NAN_2008 | MIPS_CPU_NAN_LEGACY; + } else { + c->options |= MIPS_CPU_NAN_LEGACY; + c->fpu_msk31 |= FPU_CSR_ABS2008 | FPU_CSR_NAN2008; + } + break; + case LEGACY: + c->options |= MIPS_CPU_NAN_LEGACY; + c->fpu_msk31 |= FPU_CSR_ABS2008 | FPU_CSR_NAN2008; + break; + case STD2008: + c->options |= MIPS_CPU_NAN_2008; + c->fpu_csr31 |= FPU_CSR_ABS2008 | FPU_CSR_NAN2008; + c->fpu_msk31 |= FPU_CSR_ABS2008 | FPU_CSR_NAN2008; + break; + case RELAXED: + c->options |= MIPS_CPU_NAN_2008 | MIPS_CPU_NAN_LEGACY; + break; + } +} + +/* + * Override the IEEE 754 NaN encoding and ABS.fmt/NEG.fmt execution mode + * according to the "ieee754=" parameter. + */ +static void cpu_set_nan_2008(struct cpuinfo_mips *c) +{ + switch (ieee754) { + case STRICT: + mips_use_nan_legacy = !!cpu_has_nan_legacy; + mips_use_nan_2008 = !!cpu_has_nan_2008; + break; + case LEGACY: + mips_use_nan_legacy = !!cpu_has_nan_legacy; + mips_use_nan_2008 = !cpu_has_nan_legacy; + break; + case STD2008: + mips_use_nan_legacy = !cpu_has_nan_2008; + mips_use_nan_2008 = !!cpu_has_nan_2008; + break; + case EMULATED: + /* Pretend ABS2008/NAN2008 options are dynamic */ + c->fpu_msk31 &= ~(FPU_CSR_NAN2008 | FPU_CSR_ABS2008); + fallthrough; + case RELAXED: + mips_use_nan_legacy = true; + mips_use_nan_2008 = true; + break; + } +} + +/* + * IEEE 754 NaN encoding and ABS.fmt/NEG.fmt execution mode override + * settings: + * + * strict: accept binaries that request a NaN encoding supported by the FPU + * legacy: only accept legacy-NaN binaries + * 2008: only accept 2008-NaN binaries + * relaxed: accept any binaries regardless of whether supported by the FPU + */ +static int __init ieee754_setup(char *s) +{ + if (!s) + return -1; + else if (!strcmp(s, "strict")) + ieee754 = STRICT; + else if (!strcmp(s, "emulated")) + ieee754 = EMULATED; + else if (!strcmp(s, "legacy")) + ieee754 = LEGACY; + else if (!strcmp(s, "2008")) + ieee754 = STD2008; + else if (!strcmp(s, "relaxed")) + ieee754 = RELAXED; + else + return -1; + + if (!(boot_cpu_data.options & MIPS_CPU_FPU)) + cpu_set_nofpu_2008(&boot_cpu_data); + cpu_set_nan_2008(&boot_cpu_data); + + return 0; +} + +early_param("ieee754", ieee754_setup); + +/* + * Set the FIR feature flags for the FPU emulator. + */ +static void cpu_set_nofpu_id(struct cpuinfo_mips *c) +{ + u32 value; + + value = 0; + if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 | + MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 | + MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 | + MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) + value |= MIPS_FPIR_D | MIPS_FPIR_S; + if (c->isa_level & (MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 | + MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 | + MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) + value |= MIPS_FPIR_F64 | MIPS_FPIR_L | MIPS_FPIR_W; + if (c->options & MIPS_CPU_NAN_2008) + value |= MIPS_FPIR_HAS2008; + c->fpu_id = value; +} + +/* Determined FPU emulator mask to use for the boot CPU with "nofpu". */ +static unsigned int mips_nofpu_msk31; + +/* + * Set options for FPU hardware. + */ +void cpu_set_fpu_opts(struct cpuinfo_mips *c) +{ + c->fpu_id = cpu_get_fpu_id(); + mips_nofpu_msk31 = c->fpu_msk31; + + if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 | + MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 | + MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 | + MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) { + if (c->fpu_id & MIPS_FPIR_3D) + c->ases |= MIPS_ASE_MIPS3D; + if (c->fpu_id & MIPS_FPIR_UFRP) + c->options |= MIPS_CPU_UFR; + if (c->fpu_id & MIPS_FPIR_FREP) + c->options |= MIPS_CPU_FRE; + } + + cpu_set_fpu_fcsr_mask(c); + cpu_set_fpu_2008(c); + cpu_set_nan_2008(c); +} + +/* + * Set options for the FPU emulator. + */ +void cpu_set_nofpu_opts(struct cpuinfo_mips *c) +{ + c->options &= ~MIPS_CPU_FPU; + c->fpu_msk31 = mips_nofpu_msk31; + + cpu_set_nofpu_2008(c); + cpu_set_nan_2008(c); + cpu_set_nofpu_id(c); +} + +int mips_fpu_disabled; + +static int __init fpu_disable(char *s) +{ + cpu_set_nofpu_opts(&boot_cpu_data); + mips_fpu_disabled = 1; + + return 1; +} + +__setup("nofpu", fpu_disable); + diff --git a/arch/mips/kernel/fpu-probe.h b/arch/mips/kernel/fpu-probe.h new file mode 100644 index 000000000000..951ce50890d0 --- /dev/null +++ b/arch/mips/kernel/fpu-probe.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <linux/kernel.h> + +#include <asm/cpu.h> +#include <asm/cpu-info.h> + +#ifdef CONFIG_MIPS_FP_SUPPORT + +extern int mips_fpu_disabled; + +int __cpu_has_fpu(void); +void cpu_set_fpu_opts(struct cpuinfo_mips *c); +void cpu_set_nofpu_opts(struct cpuinfo_mips *c); + +#else /* !CONFIG_MIPS_FP_SUPPORT */ + +#define mips_fpu_disabled 1 + +static inline unsigned long cpu_get_fpu_id(void) +{ + return FPIR_IMP_NONE; +} + +static inline int __cpu_has_fpu(void) +{ + return 0; +} + +static inline void cpu_set_fpu_opts(struct cpuinfo_mips *c) +{ + /* no-op */ +} + +static inline void cpu_set_nofpu_opts(struct cpuinfo_mips *c) +{ + /* no-op */ +} + +#endif /* CONFIG_MIPS_FP_SUPPORT */ diff --git a/arch/mips/kernel/ftrace.c b/arch/mips/kernel/ftrace.c index dba90ec0dc38..b15615b28569 100644 --- a/arch/mips/kernel/ftrace.c +++ b/arch/mips/kernel/ftrace.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Code for replacing ftrace calls with jumps. * @@ -11,11 +12,14 @@ #include <linux/uaccess.h> #include <linux/init.h> #include <linux/ftrace.h> +#include <linux/syscalls.h> #include <asm/asm.h> #include <asm/asm-offsets.h> #include <asm/cacheflush.h> +#include <asm/syscall.h> #include <asm/uasm.h> +#include <asm/unistd.h> #include <asm-generic/sections.h> @@ -33,24 +37,6 @@ void arch_ftrace_update_code(int command) ftrace_modify_all_code(command); } -#endif - -/* - * Check if the address is in kernel space - * - * Clone core_kernel_text() from kernel/extable.c, but doesn't call - * init_kernel_text() for Ftrace doesn't trace functions in init sections. - */ -static inline int in_kernel_space(unsigned long ip) -{ - if (ip >= (unsigned long)_stext && - ip <= (unsigned long)_etext) - return 1; - return 0; -} - -#ifdef CONFIG_DYNAMIC_FTRACE - #define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */ #define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */ #define JUMP_RANGE_MASK ((1UL << 28) - 1) @@ -60,7 +46,7 @@ static inline int in_kernel_space(unsigned long ip) ((unsigned int)(JAL | (((addr) >> 2) & ADDR_MASK))) static unsigned int insn_jal_ftrace_caller __read_mostly; -static unsigned int insn_lui_v1_hi16_mcount __read_mostly; +static unsigned int insn_la_mcount[2] __read_mostly; static unsigned int insn_j_ftrace_graph_caller __maybe_unused __read_mostly; static inline void ftrace_dyn_arch_init_insns(void) @@ -68,10 +54,20 @@ static inline void ftrace_dyn_arch_init_insns(void) u32 *buf; unsigned int v1; - /* lui v1, hi16_mcount */ - v1 = 3; - buf = (u32 *)&insn_lui_v1_hi16_mcount; - UASM_i_LA_mostly(&buf, v1, MCOUNT_ADDR); + /* If we are not in compat space, the number of generated + * instructions will exceed the maximum expected limit of 2. + * To prevent buffer overflow, we avoid generating them. + * insn_la_mcount will not be used later in ftrace_make_call. + */ + if (uasm_in_compat_space_p(MCOUNT_ADDR)) { + /* la v1, _mcount */ + v1 = 3; + buf = (u32 *)&insn_la_mcount[0]; + UASM_i_LA(&buf, v1, MCOUNT_ADDR); + } else { + pr_warn("ftrace: mcount address beyond 32 bits is not supported (%lX)\n", + MCOUNT_ADDR); + } /* jal (ftrace_caller + 8), jump over the first two instruction */ buf = (u32 *)&insn_jal_ftrace_caller; @@ -108,11 +104,35 @@ static int ftrace_modify_code_2(unsigned long ip, unsigned int new_code1, safe_store_code(new_code1, ip, faulted); if (unlikely(faulted)) return -EFAULT; + ip += 4; safe_store_code(new_code2, ip, faulted); if (unlikely(faulted)) return -EFAULT; - flush_icache_range(ip, ip + 8); /* original ip + 12 */ + + ip -= 4; + flush_icache_range(ip, ip + 8); + + return 0; +} + +static int ftrace_modify_code_2r(unsigned long ip, unsigned int new_code1, + unsigned int new_code2) +{ + int faulted; + + ip += 4; + safe_store_code(new_code2, ip, faulted); + if (unlikely(faulted)) + return -EFAULT; + + ip -= 4; + safe_store_code(new_code1, ip, faulted); + if (unlikely(faulted)) + return -EFAULT; + + flush_icache_range(ip, ip + 8); + return 0; } #endif @@ -124,13 +144,14 @@ static int ftrace_modify_code_2(unsigned long ip, unsigned int new_code1, * * move at, ra * jal _mcount --> nop + * sub sp, sp, 8 --> nop (CONFIG_32BIT) * * 2. For modules: * * 2.1 For KBUILD_MCOUNT_RA_ADDRESS and CONFIG_32BIT * * lui v1, hi_16bit_of_mcount --> b 1f (0x10000005) - * addiu v1, v1, low_16bit_of_mcount + * addiu v1, v1, low_16bit_of_mcount --> nop (CONFIG_32BIT) * move at, ra * move $12, ra_address * jalr v1 @@ -139,7 +160,7 @@ static int ftrace_modify_code_2(unsigned long ip, unsigned int new_code1, * 2.2 For the Other situations * * lui v1, hi_16bit_of_mcount --> b 1f (0x10000004) - * addiu v1, v1, low_16bit_of_mcount + * addiu v1, v1, low_16bit_of_mcount --> nop (CONFIG_32BIT) * move at, ra * jalr v1 * nop | move $12, ra_address | sub sp, sp, 8 @@ -158,7 +179,7 @@ int ftrace_make_nop(struct module *mod, * If ip is in kernel space, no long call, otherwise, long call is * needed. */ - new = in_kernel_space(ip) ? INSN_NOP : INSN_B_1F; + new = core_kernel_text(ip) ? INSN_NOP : INSN_B_1F; #ifdef CONFIG_64BIT return ftrace_modify_code(ip, new); #else @@ -178,10 +199,21 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) unsigned int new; unsigned long ip = rec->ip; - new = in_kernel_space(ip) ? insn_jal_ftrace_caller : - insn_lui_v1_hi16_mcount; + /* When the code to patch does not belong to the kernel code + * space, we must use insn_la_mcount. However, if MCOUNT_ADDR + * is not in compat space, insn_la_mcount is not usable. + */ + if (!core_kernel_text(ip) && !uasm_in_compat_space_p(MCOUNT_ADDR)) + return -EFAULT; + + new = core_kernel_text(ip) ? insn_jal_ftrace_caller : insn_la_mcount[0]; +#ifdef CONFIG_64BIT return ftrace_modify_code(ip, new); +#else + return ftrace_modify_code_2r(ip, new, core_kernel_text(ip) ? + INSN_NOP : insn_la_mcount[1]); +#endif } #define FTRACE_CALL_IP ((unsigned long)(&ftrace_call)) @@ -195,7 +227,7 @@ int ftrace_update_ftrace_func(ftrace_func_t func) return ftrace_modify_code(FTRACE_CALL_IP, new); } -int __init ftrace_dyn_arch_init(void *data) +int __init ftrace_dyn_arch_init(void) { /* Encode the instructions when booting */ ftrace_dyn_arch_init_insns(); @@ -203,9 +235,6 @@ int __init ftrace_dyn_arch_init(void *data) /* Remove "b ftrace_stub" to ensure ftrace_caller() is executed */ ftrace_modify_code(MCOUNT_ADDR, INSN_NOP); - /* The return code is retured via data */ - *(unsigned long *)data = 0; - return 0; } #endif /* CONFIG_DYNAMIC_FTRACE */ @@ -236,7 +265,7 @@ int ftrace_disable_ftrace_graph_caller(void) #define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */ #define OFFSET_MASK 0xffff /* stack offset range: 0 ~ PT_SIZE */ -unsigned long ftrace_get_parent_ra_addr(unsigned long self_ra, unsigned long +static unsigned long ftrace_get_parent_ra_addr(unsigned long self_ra, unsigned long old_parent_ra, unsigned long parent_ra_addr, unsigned long fp) { unsigned long sp, ip, tmp; @@ -248,7 +277,7 @@ unsigned long ftrace_get_parent_ra_addr(unsigned long self_ra, unsigned long * instruction "lui v1, hi_16bit_of_mcount"(offset is 24), but for * kernel, move after the instruction "move ra, at"(offset is 16) */ - ip = self_ra - (in_kernel_space(self_ra) ? 16 : 24); + ip = self_ra - (core_kernel_text(self_ra) ? 16 : 24); /* * search the text until finding the non-store instruction or "s{d,w} @@ -294,29 +323,32 @@ void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra, unsigned long fp) { unsigned long old_parent_ra; - struct ftrace_graph_ent trace; unsigned long return_hooker = (unsigned long) &return_to_handler; int faulted, insns; + if (unlikely(ftrace_graph_is_dead())) + return; + if (unlikely(atomic_read(¤t->tracing_graph_pause))) return; /* - * "parent_ra_addr" is the stack address saved the return address of - * the caller of _mcount. + * "parent_ra_addr" is the stack address where the return address of + * the caller of _mcount is saved. * - * if the gcc < 4.5, a leaf function does not save the return address - * in the stack address, so, we "emulate" one in _mcount's stack space, - * and hijack it directly, but for a non-leaf function, it save the - * return address to the its own stack space, we can not hijack it - * directly, but need to find the real stack address, - * ftrace_get_parent_addr() does it! + * If gcc < 4.5, a leaf function does not save the return address + * in the stack address, so we "emulate" one in _mcount's stack space, + * and hijack it directly. + * For a non-leaf function, it does save the return address to its own + * stack space, so we can not hijack it directly, but need to find the + * real stack address, which is done by ftrace_get_parent_addr(). * - * if gcc>= 4.5, with the new -mmcount-ra-address option, for a + * If gcc >= 4.5, with the new -mmcount-ra-address option, for a * non-leaf function, the location of the return address will be saved - * to $12 for us, and for a leaf function, only put a zero into $12. we - * do it in ftrace_graph_caller of mcount.S. + * to $12 for us. + * For a leaf function, it just puts a zero into $12, so we handle + * it in ftrace_graph_caller() of mcount.S. */ /* old_parent_ra = *parent_ra_addr; */ @@ -330,7 +362,7 @@ void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra, * If fails when getting the stack address of the non-leaf function's * ra, stop function graph tracer and return */ - if (parent_ra_addr == 0) + if (parent_ra_addr == NULL) goto out; #endif /* *parent_ra_addr = return_hooker; */ @@ -338,29 +370,50 @@ void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra, if (unlikely(faulted)) goto out; - if (ftrace_push_return_trace(old_parent_ra, self_ra, &trace.depth, fp) - == -EBUSY) { - *parent_ra_addr = old_parent_ra; - return; - } - /* * Get the recorded ip of the current mcount calling site in the * __mcount_loc section, which will be used to filter the function * entries configured through the tracing/set_graph_function interface. */ - insns = in_kernel_space(self_ra) ? 2 : MCOUNT_OFFSET_INSNS + 1; - trace.func = self_ra - (MCOUNT_INSN_SIZE * insns); + insns = core_kernel_text(self_ra) ? 2 : MCOUNT_OFFSET_INSNS + 1; + self_ra -= (MCOUNT_INSN_SIZE * insns); - /* Only trace if the calling function expects to */ - if (!ftrace_graph_entry(&trace)) { - current->curr_ret_stack--; + if (function_graph_enter(old_parent_ra, self_ra, fp, NULL)) *parent_ra_addr = old_parent_ra; - } return; out: ftrace_graph_stop(); WARN_ON(1); } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ + +#ifdef CONFIG_FTRACE_SYSCALLS + +#ifdef CONFIG_32BIT +unsigned long __init arch_syscall_addr(int nr) +{ + return (unsigned long)sys_call_table[nr - __NR_O32_Linux]; +} +#endif + +#ifdef CONFIG_64BIT + +unsigned long __init arch_syscall_addr(int nr) +{ +#ifdef CONFIG_MIPS32_N32 + if (nr >= __NR_N32_Linux && nr < __NR_N32_Linux + __NR_N32_Linux_syscalls) + return (unsigned long)sysn32_call_table[nr - __NR_N32_Linux]; +#endif + if (nr >= __NR_64_Linux && nr < __NR_64_Linux + __NR_64_Linux_syscalls) + return (unsigned long)sys_call_table[nr - __NR_64_Linux]; +#ifdef CONFIG_MIPS32_O32 + if (nr >= __NR_O32_Linux && nr < __NR_O32_Linux + __NR_O32_Linux_syscalls) + return (unsigned long)sys32_call_table[nr - __NR_O32_Linux]; +#endif + + return (unsigned long) &sys_ni_syscall; +} +#endif + +#endif /* CONFIG_FTRACE_SYSCALLS */ diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S index 31fa856829cb..be1b049d856f 100644 --- a/arch/mips/kernel/genex.S +++ b/arch/mips/kernel/genex.S @@ -18,23 +18,9 @@ #include <asm/fpregdef.h> #include <asm/mipsregs.h> #include <asm/stackframe.h> -#include <asm/war.h> +#include <asm/sync.h> #include <asm/thread_info.h> -#ifdef CONFIG_MIPS_MT_SMTC -#define PANIC_PIC(msg) \ - .set push; \ - .set nomicromips; \ - .set reorder; \ - PTR_LA a0,8f; \ - .set noat; \ - PTR_LA AT, panic; \ - jr AT; \ -9: b 9b; \ - .set pop; \ - TEXT(msg) -#endif - __INIT /* @@ -46,9 +32,6 @@ NESTED(except_vec3_generic, 0, sp) .set push .set noat -#if R5432_CP0_INTERRUPT_WAR - mfc0 k0, CP0_INDEX -#endif mfc0 k1, CP0_CAUSE andi k1, k1, 0x7c #ifdef CONFIG_64BIT @@ -67,7 +50,7 @@ NESTED(except_vec3_generic, 0, sp) */ NESTED(except_vec3_r4000, 0, sp) .set push - .set mips3 + .set arch=r4000 .set noat mfc0 k1, CP0_CAUSE li k0, 31<<2 @@ -121,50 +104,61 @@ handle_vcei: __FINIT - .align 5 /* 32 byte rollback region */ -LEAF(__r4k_wait) - .set push - .set noreorder - /* start of rollback region */ - LONG_L t0, TI_FLAGS($28) - nop - andi t0, _TIF_NEED_RESCHED - bnez t0, 1f - nop - nop - nop -#ifdef CONFIG_CPU_MICROMIPS - nop - nop - nop - nop -#endif - .set mips3 + .section .cpuidle.text,"ax" + /* Align to 32 bytes for the maximum idle interrupt region size. */ + .align 5 +LEAF(r4k_wait) + /* Keep the ISA bit clear for calculations on local labels here. */ +0: .fill 0 + /* Start of idle interrupt region. */ + local_irq_enable + /* + * If an interrupt lands here, before going idle on the next + * instruction, we must *NOT* go idle since the interrupt could + * have set TIF_NEED_RESCHED or caused a timer to need resched. + * Fall through -- see skipover_handler below -- and have the + * idle loop take care of things. + */ +1: .fill 0 + /* The R2 EI/EHB sequence takes 8 bytes, otherwise pad up. */ + .if 1b - 0b > 32 + .error "overlong idle interrupt region" + .elseif 1b - 0b > 8 + .align 4 + .endif +2: .fill 0 + .equ r4k_wait_idle_size, 2b - 0b + /* End of idle interrupt region; size has to be a power of 2. */ + .set MIPS_ISA_ARCH_LEVEL_RAW +r4k_wait_insn: wait - /* end of rollback region (the region size must be power of two) */ -1: +r4k_wait_exit: + .set mips0 + local_irq_disable jr ra - nop - .set pop - END(__r4k_wait) + END(r4k_wait) + .previous - .macro BUILD_ROLLBACK_PROLOGUE handler - FEXPORT(rollback_\handler) + .macro BUILD_SKIPOVER_PROLOGUE handler + FEXPORT(skipover_\handler) .set push .set noat MFC0 k0, CP0_EPC - PTR_LA k1, __r4k_wait - ori k0, 0x1f /* 32 byte rollback region */ - xori k0, 0x1f - bne k0, k1, 9f + /* Subtract/add 2 to let the ISA bit propagate through the mask. */ + PTR_LA k1, r4k_wait_insn - 2 + ori k0, r4k_wait_idle_size - 2 + .set noreorder + bne k0, k1, \handler + PTR_ADDIU k0, r4k_wait_exit - r4k_wait_insn + 2 + .set reorder MTC0 k0, CP0_EPC -9: .set pop .endm .align 5 -BUILD_ROLLBACK_PROLOGUE handle_int +BUILD_SKIPOVER_PROLOGUE handle_int NESTED(handle_int, PT_SIZE, sp) + .cfi_signal_frame #ifdef CONFIG_TRACE_IRQFLAGS /* * Check to see if the interrupted code has just disabled @@ -179,14 +173,14 @@ NESTED(handle_int, PT_SIZE, sp) .set push .set noat mfc0 k0, CP0_STATUS -#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX) +#if defined(CONFIG_CPU_R3000) and k0, ST0_IEP bnez k0, 1f mfc0 k0, CP0_EPC .set noreorder j k0 - rfe + rfe #else and k0, ST0_IE bnez k0, 1f @@ -196,15 +190,52 @@ NESTED(handle_int, PT_SIZE, sp) 1: .set pop #endif - SAVE_ALL + SAVE_ALL docfi=1 CLI TRACE_IRQS_OFF LONG_L s0, TI_REGS($28) LONG_S sp, TI_REGS($28) - PTR_LA ra, ret_from_irq - PTR_LA v0, plat_irq_dispatch - jr v0 + + /* + * SAVE_ALL ensures we are using a valid kernel stack for the thread. + * Check if we are already using the IRQ stack. + */ + move s1, sp # Preserve the sp + + /* Get IRQ stack for this CPU */ + ASM_CPUID_MFC0 k0, ASM_SMP_CPUID_REG +#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32) + lui k1, %hi(irq_stack) +#else + lui k1, %highest(irq_stack) + daddiu k1, %higher(irq_stack) + dsll k1, 16 + daddiu k1, %hi(irq_stack) + dsll k1, 16 +#endif + LONG_SRL k0, SMP_CPUID_PTRSHIFT + LONG_ADDU k1, k0 + LONG_L t0, %lo(irq_stack)(k1) + + # Check if already on IRQ stack + PTR_LI t1, ~(_THREAD_SIZE-1) + and t1, t1, sp + beq t0, t1, 2f + + /* Switch to IRQ stack */ + li t1, _IRQ_STACK_START + PTR_ADD sp, t0, t1 + + /* Save task's sp on IRQ stack so that unwinding can follow it */ + LONG_S s1, 0(sp) +2: + jal plat_irq_dispatch + + /* Restore sp */ + move sp, s1 + + j ret_from_irq #ifdef CONFIG_CPU_MICROMIPS nop #endif @@ -245,76 +276,78 @@ NESTED(except_vec_ejtag_debug, 0, sp) * This prototype is copied to ebase + n*IntCtl.VS and patched * to invoke the handler */ -BUILD_ROLLBACK_PROLOGUE except_vec_vi +BUILD_SKIPOVER_PROLOGUE except_vec_vi NESTED(except_vec_vi, 0, sp) - SAVE_SOME - SAVE_AT + SAVE_SOME docfi=1 + SAVE_AT docfi=1 .set push .set noreorder -#ifdef CONFIG_MIPS_MT_SMTC - /* - * To keep from blindly blocking *all* interrupts - * during service by SMTC kernel, we also want to - * pass the IM value to be cleared. - */ -FEXPORT(except_vec_vi_mori) - ori a0, $0, 0 -#endif /* CONFIG_MIPS_MT_SMTC */ PTR_LA v1, except_vec_vi_handler -FEXPORT(except_vec_vi_lui) - lui v0, 0 /* Patched */ jr v1 FEXPORT(except_vec_vi_ori) - ori v0, 0 /* Patched */ + ori v0, zero, 0 /* Offset in vi_handlers[] */ .set pop END(except_vec_vi) EXPORT(except_vec_vi_end) /* * Common Vectored Interrupt code - * Complete the register saves and invoke the handler which is passed in $v0 + * Complete the register saves and invoke the handler, $v0 holds + * offset into vi_handlers[] */ NESTED(except_vec_vi_handler, 0, sp) SAVE_TEMP SAVE_STATIC -#ifdef CONFIG_MIPS_MT_SMTC - /* - * SMTC has an interesting problem that interrupts are level-triggered, - * and the CLI macro will clear EXL, potentially causing a duplicate - * interrupt service invocation. So we need to clear the associated - * IM bit of Status prior to doing CLI, and restore it after the - * service routine has been invoked - we must assume that the - * service routine will have cleared the state, and any active - * level represents a new or otherwised unserviced event... - */ - mfc0 t1, CP0_STATUS - and t0, a0, t1 -#ifdef CONFIG_MIPS_MT_SMTC_IM_BACKSTOP - mfc0 t2, CP0_TCCONTEXT - or t2, t0, t2 - mtc0 t2, CP0_TCCONTEXT -#endif /* CONFIG_MIPS_MT_SMTC_IM_BACKSTOP */ - xor t1, t1, t0 - mtc0 t1, CP0_STATUS - _ehb -#endif /* CONFIG_MIPS_MT_SMTC */ CLI #ifdef CONFIG_TRACE_IRQFLAGS move s0, v0 -#ifdef CONFIG_MIPS_MT_SMTC - move s1, a0 -#endif TRACE_IRQS_OFF -#ifdef CONFIG_MIPS_MT_SMTC - move a0, s1 -#endif move v0, s0 #endif LONG_L s0, TI_REGS($28) LONG_S sp, TI_REGS($28) - PTR_LA ra, ret_from_irq - jr v0 + + /* + * SAVE_ALL ensures we are using a valid kernel stack for the thread. + * Check if we are already using the IRQ stack. + */ + move s1, sp # Preserve the sp + + /* Get IRQ stack for this CPU */ + ASM_CPUID_MFC0 k0, ASM_SMP_CPUID_REG +#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32) + lui k1, %hi(irq_stack) +#else + lui k1, %highest(irq_stack) + daddiu k1, %higher(irq_stack) + dsll k1, 16 + daddiu k1, %hi(irq_stack) + dsll k1, 16 +#endif + LONG_SRL k0, SMP_CPUID_PTRSHIFT + LONG_ADDU k1, k0 + LONG_L t0, %lo(irq_stack)(k1) + + # Check if already on IRQ stack + PTR_LI t1, ~(_THREAD_SIZE-1) + and t1, t1, sp + beq t0, t1, 2f + + /* Switch to IRQ stack */ + li t1, _IRQ_STACK_START + PTR_ADD sp, t0, t1 + + /* Save task's sp on IRQ stack so that unwinding can follow it */ + LONG_S s1, 0(sp) +2: + PTR_L v0, vi_handlers(v0) + jalr v0 + + /* Restore sp */ + move sp, s1 + + j ret_from_irq END(except_vec_vi_handler) /* @@ -326,23 +359,64 @@ NESTED(ejtag_debug_handler, PT_SIZE, sp) MTC0 k0, CP0_DESAVE mfc0 k0, CP0_DEBUG - sll k0, k0, 30 # Check for SDBBP. - bgez k0, ejtag_return + andi k0, k0, MIPS_DEBUG_DBP # Check for SDBBP. + beqz k0, ejtag_return + +#ifdef CONFIG_SMP +1: PTR_LA k0, ejtag_debug_buffer_spinlock + __SYNC(full, loongson3_war) +2: ll k0, 0(k0) + bnez k0, 2b + PTR_LA k0, ejtag_debug_buffer_spinlock + sc k0, 0(k0) + beqz k0, 1b +# ifdef CONFIG_WEAK_REORDERING_BEYOND_LLSC + sync +# endif PTR_LA k0, ejtag_debug_buffer LONG_S k1, 0(k0) + + ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG + PTR_SRL k1, SMP_CPUID_PTRSHIFT + PTR_SLL k1, LONGLOG + PTR_LA k0, ejtag_debug_buffer_per_cpu + PTR_ADDU k0, k1 + + PTR_LA k1, ejtag_debug_buffer + LONG_L k1, 0(k1) + LONG_S k1, 0(k0) + + PTR_LA k0, ejtag_debug_buffer_spinlock + sw zero, 0(k0) +#else + PTR_LA k0, ejtag_debug_buffer + LONG_S k1, 0(k0) +#endif + SAVE_ALL move a0, sp jal ejtag_exception_handler RESTORE_ALL + +#ifdef CONFIG_SMP + ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG + PTR_SRL k1, SMP_CPUID_PTRSHIFT + PTR_SLL k1, LONGLOG + PTR_LA k0, ejtag_debug_buffer_per_cpu + PTR_ADDU k0, k1 + LONG_L k1, 0(k0) +#else PTR_LA k0, ejtag_debug_buffer LONG_L k1, 0(k0) +#endif ejtag_return: + back_to_back_c0_hazard MFC0 k0, CP0_DESAVE .set mips32 deret - .set pop + .set pop END(ejtag_debug_handler) /* @@ -352,6 +426,12 @@ ejtag_return: .data EXPORT(ejtag_debug_buffer) .fill LONGSIZE +#ifdef CONFIG_SMP +EXPORT(ejtag_debug_buffer_spinlock) + .fill LONGSIZE +EXPORT(ejtag_debug_buffer_per_cpu) + .fill LONGSIZE * NR_CPUS +#endif .previous __INIT @@ -372,14 +452,23 @@ NESTED(except_vec_nmi, 0, sp) __FINIT NESTED(nmi_handler, PT_SIZE, sp) + .cfi_signal_frame .set push .set noat + /* + * Clear ERL - restore segment mapping + * Clear BEV - required for page fault exception handler to work + */ + mfc0 k0, CP0_STATUS + ori k0, k0, ST0_EXL + li k1, ~(ST0_BEV | ST0_ERL) + and k0, k0, k1 + mtc0 k0, CP0_STATUS + _ehb SAVE_ALL move a0, sp jal nmi_exception_handler - RESTORE_ALL - .set mips3 - eret + /* nmi_exception_handler never returns */ .set pop END(nmi_handler) @@ -397,16 +486,20 @@ NESTED(nmi_handler, PT_SIZE, sp) .endm .macro __build_clear_fpe + CLI + TRACE_IRQS_OFF .set push /* gas fails to assemble cfc1 for some archs (octeon).*/ \ .set mips1 + .set hardfloat cfc1 a1, fcr31 - li a2, ~(0x3f << 12) - and a2, a1 - ctc1 a2, fcr31 .set pop - TRACE_IRQS_ON - STI + .endm + + .macro __build_clear_msa_fpe + CLI + TRACE_IRQS_OFF + _cfcmsa a1, MSA_CSR .endm .macro __build_clear_ade @@ -415,26 +508,39 @@ NESTED(nmi_handler, PT_SIZE, sp) KMODE .endm + .macro __build_clear_gsexc + .set push + /* + * We need to specify a selector to access the CP0.Diag1 (GSCause) + * register. All GSExc-equipped processors have MIPS32. + */ + .set mips32 + mfc0 a1, CP0_DIAGNOSTIC1 + .set pop + TRACE_IRQS_ON + STI + .endm + .macro __BUILD_silent exception .endm - /* Gas tries to parse the PRINT argument as a string containing + /* Gas tries to parse the ASM_PRINT argument as a string containing string escapes and emits bogus warnings if it believes to recognize an unknown escape code. So make the arguments start with an n and gas will believe \n is ok ... */ .macro __BUILD_verbose nexception LONG_L a1, PT_EPC(sp) #ifdef CONFIG_32BIT - PRINT("Got \nexception at %08lx\012") + ASM_PRINT("Got \nexception at %08lx\012") #endif #ifdef CONFIG_64BIT - PRINT("Got \nexception at %016lx\012") + ASM_PRINT("Got \nexception at %016lx\012") #endif .endm .macro __BUILD_count exception LONG_L t0,exception_count_\exception - LONG_ADDIU t0, 1 + LONG_ADDIU t0, 1 LONG_S t0,exception_count_\exception .comm exception_count\exception, 8, 8 .endm @@ -442,15 +548,16 @@ NESTED(nmi_handler, PT_SIZE, sp) .macro __BUILD_HANDLER exception handler clear verbose ext .align 5 NESTED(handle_\exception, PT_SIZE, sp) + .cfi_signal_frame .set noat SAVE_ALL FEXPORT(handle_\exception\ext) - __BUILD_clear_\clear + __build_clear_\clear .set at __BUILD_\verbose \exception move a0, sp - PTR_LA ra, ret_from_exception - j do_\handler + jal do_\handler + j ret_from_exception END(handle_\exception) .endm @@ -467,7 +574,13 @@ NESTED(nmi_handler, PT_SIZE, sp) BUILD_HANDLER cpu cpu sti silent /* #11 */ BUILD_HANDLER ov ov sti silent /* #12 */ BUILD_HANDLER tr tr sti silent /* #13 */ + BUILD_HANDLER msa_fpe msa_fpe msa_fpe silent /* #14 */ +#ifdef CONFIG_MIPS_FP_SUPPORT BUILD_HANDLER fpe fpe fpe silent /* #15 */ +#endif + BUILD_HANDLER ftlb ftlb none silent /* #16 */ + BUILD_HANDLER gsexc gsexc gsexc silent /* #16 */ + BUILD_HANDLER msa msa sti silent /* #21 */ BUILD_HANDLER mdmx mdmx sti silent /* #22 */ #ifdef CONFIG_HARDWARE_WATCHPOINTS /* @@ -484,19 +597,16 @@ NESTED(nmi_handler, PT_SIZE, sp) BUILD_HANDLER reserved reserved sti verbose /* others */ .align 5 - LEAF(handle_ri_rdhwr_vivt) -#ifdef CONFIG_MIPS_MT_SMTC - PANIC_PIC("handle_ri_rdhwr_vivt called") -#else + LEAF(handle_ri_rdhwr_tlbp) .set push .set noat .set noreorder /* check if TLB contains a entry for EPC */ MFC0 k1, CP0_ENTRYHI - andi k1, 0xff /* ASID_MASK */ + andi k1, MIPS_ENTRYHI_ASID | MIPS_ENTRYHI_ASIDX MFC0 k0, CP0_EPC - PTR_SRL k0, _PAGE_SHIFT + 1 - PTR_SLL k0, _PAGE_SHIFT + 1 + PTR_SRL k0, _PAGE_SHIFT + 1 + PTR_SLL k0, _PAGE_SHIFT + 1 or k1, k0 MTC0 k1, CP0_ENTRYHI mtc0_tlbw_hazard @@ -506,8 +616,7 @@ NESTED(nmi_handler, PT_SIZE, sp) .set pop bltz k1, handle_ri /* slow path */ /* fall thru */ -#endif - END(handle_ri_rdhwr_vivt) + END(handle_ri_rdhwr_tlbp) LEAF(handle_ri_rdhwr) .set push @@ -517,27 +626,27 @@ NESTED(nmi_handler, PT_SIZE, sp) /* microMIPS: 0x007d6b3c: rdhwr v1,$29 */ MFC0 k1, CP0_EPC #if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_MIPS32_R2) || defined(CONFIG_CPU_MIPS64_R2) - and k0, k1, 1 - beqz k0, 1f - xor k1, k0 - lhu k0, (k1) - lhu k1, 2(k1) - ins k1, k0, 16, 16 - lui k0, 0x007d - b docheck - ori k0, 0x6b3c + and k0, k1, 1 + beqz k0, 1f + xor k1, k0 + lhu k0, (k1) + lhu k1, 2(k1) + ins k1, k0, 16, 16 + lui k0, 0x007d + b docheck + ori k0, 0x6b3c 1: - lui k0, 0x7c03 - lw k1, (k1) - ori k0, 0xe83b + lui k0, 0x7c03 + lw k1, (k1) + ori k0, 0xe83b #else - andi k0, k1, 1 - bnez k0, handle_ri - lui k0, 0x7c03 - lw k1, (k1) - ori k0, 0xe83b + andi k0, k1, 1 + bnez k0, handle_ri + lui k0, 0x7c03 + lw k1, (k1) + ori k0, 0xe83b #endif - .set reorder + .set reorder docheck: bne k0, k1, handle_ri /* if not ours */ @@ -546,7 +655,7 @@ isrdhwr: get_saved_sp /* k1 := current_thread_info */ .set noreorder MFC0 k0, CP0_EPC -#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX) +#if defined(CONFIG_CPU_R3000) ori k1, _THREAD_MASK xori k1, _THREAD_MASK LONG_L v1, TI_TP_VALUE(k1) @@ -566,14 +675,15 @@ isrdhwr: ori k1, _THREAD_MASK xori k1, _THREAD_MASK LONG_L v1, TI_TP_VALUE(k1) - .set mips3 + .set push + .set arch=r4000 eret - .set mips0 + .set pop #endif .set pop END(handle_ri_rdhwr) -#ifdef CONFIG_64BIT +#ifdef CONFIG_CPU_R4X00_BUGS64 /* A temporary overflow handler used by check_daddi(). */ __INIT diff --git a/arch/mips/kernel/gpio_txx9.c b/arch/mips/kernel/gpio_txx9.c index c6854d9df926..96ac40d20c23 100644 --- a/arch/mips/kernel/gpio_txx9.c +++ b/arch/mips/kernel/gpio_txx9.c @@ -1,16 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * A gpio chip driver for TXx9 SoCs * * Copyright (C) 2008 Atsushi Nemoto <anemo@mba.ocn.ne.jp> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include <linux/init.h> #include <linux/spinlock.h> -#include <linux/gpio.h> +#include <linux/gpio/driver.h> #include <linux/errno.h> #include <linux/io.h> #include <asm/txx9pio.h> @@ -21,7 +18,7 @@ static struct txx9_pio_reg __iomem *txx9_pioptr; static int txx9_gpio_get(struct gpio_chip *chip, unsigned int offset) { - return __raw_readl(&txx9_pioptr->din) & (1 << offset); + return !!(__raw_readl(&txx9_pioptr->din) & (1 << offset)); } static void txx9_gpio_set_raw(unsigned int offset, int value) @@ -35,14 +32,16 @@ static void txx9_gpio_set_raw(unsigned int offset, int value) __raw_writel(val, &txx9_pioptr->dout); } -static void txx9_gpio_set(struct gpio_chip *chip, unsigned int offset, - int value) +static int txx9_gpio_set(struct gpio_chip *chip, unsigned int offset, + int value) { unsigned long flags; spin_lock_irqsave(&txx9_gpio_lock, flags); txx9_gpio_set_raw(offset, value); mmiowb(); spin_unlock_irqrestore(&txx9_gpio_lock, flags); + + return 0; } static int txx9_gpio_dir_in(struct gpio_chip *chip, unsigned int offset) @@ -85,5 +84,5 @@ int __init txx9_gpio_init(unsigned long baseaddr, return -ENODEV; txx9_gpio_chip.base = base; txx9_gpio_chip.ngpio = num; - return gpiochip_add(&txx9_gpio_chip); + return gpiochip_add_data(&txx9_gpio_chip, NULL); } diff --git a/arch/mips/kernel/head.S b/arch/mips/kernel/head.S index 099912324423..d99ed58b7043 100644 --- a/arch/mips/kernel/head.S +++ b/arch/mips/kernel/head.S @@ -21,7 +21,6 @@ #include <asm/asmmacro.h> #include <asm/irqflags.h> #include <asm/regdef.h> -#include <asm/pgtable-bits.h> #include <asm/mipsregs.h> #include <asm/stackframe.h> @@ -35,33 +34,12 @@ */ .macro setup_c0_status set clr .set push -#ifdef CONFIG_MIPS_MT_SMTC - /* - * For SMTC, we need to set privilege and disable interrupts only for - * the current TC, using the TCStatus register. - */ - mfc0 t0, CP0_TCSTATUS - /* Fortunately CU 0 is in the same place in both registers */ - /* Set TCU0, TMX, TKSU (for later inversion) and IXMT */ - li t1, ST0_CU0 | 0x08001c00 - or t0, t1 - /* Clear TKSU, leave IXMT */ - xori t0, 0x00001800 - mtc0 t0, CP0_TCSTATUS - _ehb - /* We need to leave the global IE bit set, but clear EXL...*/ - mfc0 t0, CP0_STATUS - or t0, ST0_CU0 | ST0_EXL | ST0_ERL | \set | \clr - xor t0, ST0_EXL | ST0_ERL | \clr - mtc0 t0, CP0_STATUS -#else mfc0 t0, CP0_STATUS - or t0, ST0_CU0|\set|0x1f|\clr + or t0, ST0_KERNEL_CUMASK|\set|0x1f|\clr xor t0, 0x1f|\clr mtc0 t0, CP0_STATUS .set noreorder sll zero,3 # ehb -#endif .set pop .endm @@ -81,6 +59,8 @@ #endif .endm + __HEAD + #ifndef CONFIG_NO_EXCEPT_FILL /* * Reserved space for exception handlers. @@ -99,7 +79,7 @@ EXPORT(_stext) */ FEXPORT(__kernel_entry) j kernel_entry -#endif +#endif /* CONFIG_BOOT_RAW */ __REF @@ -115,24 +95,6 @@ NESTED(kernel_entry, 16, sp) # kernel entry point jr t0 0: -#ifdef CONFIG_MIPS_MT_SMTC - /* - * In SMTC kernel, "CLI" is thread-specific, in TCStatus. - * We still need to enable interrupts globally in Status, - * and clear EXL/ERL. - * - * TCContext is used to track interrupt levels under - * service in SMTC kernel. Clear for boot TC before - * allowing any interrupts. - */ - mtc0 zero, CP0_TCCONTEXT - - mfc0 t0, CP0_STATUS - ori t0, t0, 0xff1f - xori t0, t0, 0x001e - mtc0 t0, CP0_STATUS -#endif /* CONFIG_MIPS_MT_SMTC */ - PTR_LA t0, __bss_start # clear .bss LONG_S zero, (t0) PTR_LA t1, __bss_stop - LONGSIZE @@ -147,6 +109,9 @@ NESTED(kernel_entry, 16, sp) # kernel entry point LONG_S a3, fw_arg3 MTC0 zero, CP0_CONTEXT # clear context register +#ifdef CONFIG_64BIT + MTC0 zero, CP0_XCONTEXT +#endif PTR_LA $28, init_thread_union /* Set the SP after an empty pt_regs. */ PTR_LI sp, _THREAD_SIZE - 32 - PT_SIZE @@ -155,38 +120,37 @@ NESTED(kernel_entry, 16, sp) # kernel entry point set_saved_sp sp, t0, t1 PTR_SUBU sp, 4 * SZREG # init stack pointer +#ifdef CONFIG_RELOCATABLE + /* Copy kernel and apply the relocations */ + jal relocate_kernel + + /* Repoint the sp into the new kernel image */ + PTR_LI sp, _THREAD_SIZE - 32 - PT_SIZE + PTR_ADDU sp, $28 + set_saved_sp sp, t0, t1 + PTR_SUBU sp, 4 * SZREG # init stack pointer + + /* + * relocate_kernel returns the entry point either + * in the relocated kernel or the original if for + * some reason relocation failed - jump there now + * with instruction hazard barrier because of the + * newly sync'd icache. + */ + jr.hb v0 +#else /* !CONFIG_RELOCATABLE */ j start_kernel +#endif /* !CONFIG_RELOCATABLE */ END(kernel_entry) - __CPUINIT - #ifdef CONFIG_SMP /* * SMP slave cpus entry point. Board specific code for bootstrap calls this * function after setting up the stack and gp registers. */ NESTED(smp_bootstrap, 16, sp) -#ifdef CONFIG_MIPS_MT_SMTC - /* - * Read-modify-writes of Status must be atomic, and this - * is one case where CLI is invoked without EXL being - * necessarily set. The CLI and setup_c0_status will - * in fact be redundant for all but the first TC of - * each VPE being booted. - */ - DMT 10 # dmt t2 /* t0, t1 are used by CLI and setup_c0_status() */ - jal mips_ihb -#endif /* CONFIG_MIPS_MT_SMTC */ - setup_c0_status_sec smp_slave_setup -#ifdef CONFIG_MIPS_MT_SMTC - andi t2, t2, VPECONTROL_TE - beqz t2, 2f - EMT # emt -2: -#endif /* CONFIG_MIPS_MT_SMTC */ + setup_c0_status_sec j start_secondary END(smp_bootstrap) #endif /* CONFIG_SMP */ - - __FINIT diff --git a/arch/mips/kernel/i8253.c b/arch/mips/kernel/i8253.c index c5bc344fc745..ca21210e06b5 100644 --- a/arch/mips/kernel/i8253.c +++ b/arch/mips/kernel/i8253.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * i8253.c 8253/PIT functions * @@ -17,21 +18,19 @@ static irqreturn_t timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static struct irqaction irq0 = { - .handler = timer_interrupt, - .flags = IRQF_NOBALANCING | IRQF_TIMER, - .name = "timer" -}; - void __init setup_pit_timer(void) { + unsigned long flags = IRQF_NOBALANCING | IRQF_TIMER; + clockevent_i8253_init(true); - setup_irq(0, &irq0); + if (request_irq(0, timer_interrupt, flags, "timer", NULL)) + pr_err("Failed to request irq 0 (timer)\n"); } static int __init init_pit_clocksource(void) { - if (num_possible_cpus() > 1) /* PIT does not scale! */ + if (num_possible_cpus() > 1 || /* PIT does not scale! */ + !clockevent_state_periodic(&i8253_clockevent)) return 0; return clocksource_i8253_init(); diff --git a/arch/mips/kernel/i8259.c b/arch/mips/kernel/i8259.c deleted file mode 100644 index 2b91fe80c436..000000000000 --- a/arch/mips/kernel/i8259.c +++ /dev/null @@ -1,335 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Code to handle x86 style IRQs plus some generic interrupt stuff. - * - * Copyright (C) 1992 Linus Torvalds - * Copyright (C) 1994 - 2000 Ralf Baechle - */ -#include <linux/delay.h> -#include <linux/init.h> -#include <linux/ioport.h> -#include <linux/interrupt.h> -#include <linux/kernel.h> -#include <linux/spinlock.h> -#include <linux/syscore_ops.h> -#include <linux/irq.h> - -#include <asm/i8259.h> -#include <asm/io.h> - -/* - * This is the 'legacy' 8259A Programmable Interrupt Controller, - * present in the majority of PC/AT boxes. - * plus some generic x86 specific things if generic specifics makes - * any sense at all. - * this file should become arch/i386/kernel/irq.c when the old irq.c - * moves to arch independent land - */ - -static int i8259A_auto_eoi = -1; -DEFINE_RAW_SPINLOCK(i8259A_lock); -static void disable_8259A_irq(struct irq_data *d); -static void enable_8259A_irq(struct irq_data *d); -static void mask_and_ack_8259A(struct irq_data *d); -static void init_8259A(int auto_eoi); - -static struct irq_chip i8259A_chip = { - .name = "XT-PIC", - .irq_mask = disable_8259A_irq, - .irq_disable = disable_8259A_irq, - .irq_unmask = enable_8259A_irq, - .irq_mask_ack = mask_and_ack_8259A, -#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF - .irq_set_affinity = plat_set_irq_affinity, -#endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */ -}; - -/* - * 8259A PIC functions to handle ISA devices: - */ - -/* - * This contains the irq mask for both 8259A irq controllers, - */ -static unsigned int cached_irq_mask = 0xffff; - -#define cached_master_mask (cached_irq_mask) -#define cached_slave_mask (cached_irq_mask >> 8) - -static void disable_8259A_irq(struct irq_data *d) -{ - unsigned int mask, irq = d->irq - I8259A_IRQ_BASE; - unsigned long flags; - - mask = 1 << irq; - raw_spin_lock_irqsave(&i8259A_lock, flags); - cached_irq_mask |= mask; - if (irq & 8) - outb(cached_slave_mask, PIC_SLAVE_IMR); - else - outb(cached_master_mask, PIC_MASTER_IMR); - raw_spin_unlock_irqrestore(&i8259A_lock, flags); -} - -static void enable_8259A_irq(struct irq_data *d) -{ - unsigned int mask, irq = d->irq - I8259A_IRQ_BASE; - unsigned long flags; - - mask = ~(1 << irq); - raw_spin_lock_irqsave(&i8259A_lock, flags); - cached_irq_mask &= mask; - if (irq & 8) - outb(cached_slave_mask, PIC_SLAVE_IMR); - else - outb(cached_master_mask, PIC_MASTER_IMR); - raw_spin_unlock_irqrestore(&i8259A_lock, flags); -} - -int i8259A_irq_pending(unsigned int irq) -{ - unsigned int mask; - unsigned long flags; - int ret; - - irq -= I8259A_IRQ_BASE; - mask = 1 << irq; - raw_spin_lock_irqsave(&i8259A_lock, flags); - if (irq < 8) - ret = inb(PIC_MASTER_CMD) & mask; - else - ret = inb(PIC_SLAVE_CMD) & (mask >> 8); - raw_spin_unlock_irqrestore(&i8259A_lock, flags); - - return ret; -} - -void make_8259A_irq(unsigned int irq) -{ - disable_irq_nosync(irq); - irq_set_chip_and_handler(irq, &i8259A_chip, handle_level_irq); - enable_irq(irq); -} - -/* - * This function assumes to be called rarely. Switching between - * 8259A registers is slow. - * This has to be protected by the irq controller spinlock - * before being called. - */ -static inline int i8259A_irq_real(unsigned int irq) -{ - int value; - int irqmask = 1 << irq; - - if (irq < 8) { - outb(0x0B, PIC_MASTER_CMD); /* ISR register */ - value = inb(PIC_MASTER_CMD) & irqmask; - outb(0x0A, PIC_MASTER_CMD); /* back to the IRR register */ - return value; - } - outb(0x0B, PIC_SLAVE_CMD); /* ISR register */ - value = inb(PIC_SLAVE_CMD) & (irqmask >> 8); - outb(0x0A, PIC_SLAVE_CMD); /* back to the IRR register */ - return value; -} - -/* - * Careful! The 8259A is a fragile beast, it pretty - * much _has_ to be done exactly like this (mask it - * first, _then_ send the EOI, and the order of EOI - * to the two 8259s is important! - */ -static void mask_and_ack_8259A(struct irq_data *d) -{ - unsigned int irqmask, irq = d->irq - I8259A_IRQ_BASE; - unsigned long flags; - - irqmask = 1 << irq; - raw_spin_lock_irqsave(&i8259A_lock, flags); - /* - * Lightweight spurious IRQ detection. We do not want - * to overdo spurious IRQ handling - it's usually a sign - * of hardware problems, so we only do the checks we can - * do without slowing down good hardware unnecessarily. - * - * Note that IRQ7 and IRQ15 (the two spurious IRQs - * usually resulting from the 8259A-1|2 PICs) occur - * even if the IRQ is masked in the 8259A. Thus we - * can check spurious 8259A IRQs without doing the - * quite slow i8259A_irq_real() call for every IRQ. - * This does not cover 100% of spurious interrupts, - * but should be enough to warn the user that there - * is something bad going on ... - */ - if (cached_irq_mask & irqmask) - goto spurious_8259A_irq; - cached_irq_mask |= irqmask; - -handle_real_irq: - if (irq & 8) { - inb(PIC_SLAVE_IMR); /* DUMMY - (do we need this?) */ - outb(cached_slave_mask, PIC_SLAVE_IMR); - outb(0x60+(irq&7), PIC_SLAVE_CMD);/* 'Specific EOI' to slave */ - outb(0x60+PIC_CASCADE_IR, PIC_MASTER_CMD); /* 'Specific EOI' to master-IRQ2 */ - } else { - inb(PIC_MASTER_IMR); /* DUMMY - (do we need this?) */ - outb(cached_master_mask, PIC_MASTER_IMR); - outb(0x60+irq, PIC_MASTER_CMD); /* 'Specific EOI to master */ - } - smtc_im_ack_irq(irq); - raw_spin_unlock_irqrestore(&i8259A_lock, flags); - return; - -spurious_8259A_irq: - /* - * this is the slow path - should happen rarely. - */ - if (i8259A_irq_real(irq)) - /* - * oops, the IRQ _is_ in service according to the - * 8259A - not spurious, go handle it. - */ - goto handle_real_irq; - - { - static int spurious_irq_mask; - /* - * At this point we can be sure the IRQ is spurious, - * lets ACK and report it. [once per IRQ] - */ - if (!(spurious_irq_mask & irqmask)) { - printk(KERN_DEBUG "spurious 8259A interrupt: IRQ%d.\n", irq); - spurious_irq_mask |= irqmask; - } - atomic_inc(&irq_err_count); - /* - * Theoretically we do not have to handle this IRQ, - * but in Linux this does not cause problems and is - * simpler for us. - */ - goto handle_real_irq; - } -} - -static void i8259A_resume(void) -{ - if (i8259A_auto_eoi >= 0) - init_8259A(i8259A_auto_eoi); -} - -static void i8259A_shutdown(void) -{ - /* Put the i8259A into a quiescent state that - * the kernel initialization code can get it - * out of. - */ - if (i8259A_auto_eoi >= 0) { - outb(0xff, PIC_MASTER_IMR); /* mask all of 8259A-1 */ - outb(0xff, PIC_SLAVE_IMR); /* mask all of 8259A-2 */ - } -} - -static struct syscore_ops i8259_syscore_ops = { - .resume = i8259A_resume, - .shutdown = i8259A_shutdown, -}; - -static int __init i8259A_init_sysfs(void) -{ - register_syscore_ops(&i8259_syscore_ops); - return 0; -} - -device_initcall(i8259A_init_sysfs); - -static void init_8259A(int auto_eoi) -{ - unsigned long flags; - - i8259A_auto_eoi = auto_eoi; - - raw_spin_lock_irqsave(&i8259A_lock, flags); - - outb(0xff, PIC_MASTER_IMR); /* mask all of 8259A-1 */ - outb(0xff, PIC_SLAVE_IMR); /* mask all of 8259A-2 */ - - /* - * outb_p - this has to work on a wide range of PC hardware. - */ - outb_p(0x11, PIC_MASTER_CMD); /* ICW1: select 8259A-1 init */ - outb_p(I8259A_IRQ_BASE + 0, PIC_MASTER_IMR); /* ICW2: 8259A-1 IR0 mapped to I8259A_IRQ_BASE + 0x00 */ - outb_p(1U << PIC_CASCADE_IR, PIC_MASTER_IMR); /* 8259A-1 (the master) has a slave on IR2 */ - if (auto_eoi) /* master does Auto EOI */ - outb_p(MASTER_ICW4_DEFAULT | PIC_ICW4_AEOI, PIC_MASTER_IMR); - else /* master expects normal EOI */ - outb_p(MASTER_ICW4_DEFAULT, PIC_MASTER_IMR); - - outb_p(0x11, PIC_SLAVE_CMD); /* ICW1: select 8259A-2 init */ - outb_p(I8259A_IRQ_BASE + 8, PIC_SLAVE_IMR); /* ICW2: 8259A-2 IR0 mapped to I8259A_IRQ_BASE + 0x08 */ - outb_p(PIC_CASCADE_IR, PIC_SLAVE_IMR); /* 8259A-2 is a slave on master's IR2 */ - outb_p(SLAVE_ICW4_DEFAULT, PIC_SLAVE_IMR); /* (slave's support for AEOI in flat mode is to be investigated) */ - if (auto_eoi) - /* - * In AEOI mode we just have to mask the interrupt - * when acking. - */ - i8259A_chip.irq_mask_ack = disable_8259A_irq; - else - i8259A_chip.irq_mask_ack = mask_and_ack_8259A; - - udelay(100); /* wait for 8259A to initialize */ - - outb(cached_master_mask, PIC_MASTER_IMR); /* restore master IRQ mask */ - outb(cached_slave_mask, PIC_SLAVE_IMR); /* restore slave IRQ mask */ - - raw_spin_unlock_irqrestore(&i8259A_lock, flags); -} - -/* - * IRQ2 is cascade interrupt to second interrupt controller - */ -static struct irqaction irq2 = { - .handler = no_action, - .name = "cascade", - .flags = IRQF_NO_THREAD, -}; - -static struct resource pic1_io_resource = { - .name = "pic1", - .start = PIC_MASTER_CMD, - .end = PIC_MASTER_IMR, - .flags = IORESOURCE_BUSY -}; - -static struct resource pic2_io_resource = { - .name = "pic2", - .start = PIC_SLAVE_CMD, - .end = PIC_SLAVE_IMR, - .flags = IORESOURCE_BUSY -}; - -/* - * On systems with i8259-style interrupt controllers we assume for - * driver compatibility reasons interrupts 0 - 15 to be the i8259 - * interrupts even if the hardware uses a different interrupt numbering. - */ -void __init init_i8259_irqs(void) -{ - int i; - - insert_resource(&ioport_resource, &pic1_io_resource); - insert_resource(&ioport_resource, &pic2_io_resource); - - init_8259A(0); - - for (i = I8259A_IRQ_BASE; i < I8259A_IRQ_BASE + 16; i++) { - irq_set_chip_and_handler(i, &i8259A_chip, handle_level_irq); - irq_set_probe(i); - } - - setup_irq(I8259A_IRQ_BASE + PIC_CASCADE_IR, &irq2); -} diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c index 0c655deeea4a..80e8a04a642e 100644 --- a/arch/mips/kernel/idle.c +++ b/arch/mips/kernel/idle.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * MIPS idle loop and WAIT instruction support. * @@ -5,12 +6,8 @@ * Copyright (C) 1994 - 2006 Ralf Baechle * Copyright (C) 2003, 2004 Maciej W. Rozycki * Copyright (C) 2001, 2004, 2011, 2012 MIPS Technologies, Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ +#include <linux/cpu.h> #include <linux/export.h> #include <linux/init.h> #include <linux/irqflags.h> @@ -18,6 +15,7 @@ #include <linux/sched.h> #include <asm/cpu.h> #include <asm/cpu-info.h> +#include <asm/cpu-type.h> #include <asm/idle.h> #include <asm/mipsregs.h> @@ -31,24 +29,10 @@ void (*cpu_wait)(void); EXPORT_SYMBOL(cpu_wait); -static void r3081_wait(void) +static void __cpuidle r3081_wait(void) { unsigned long cfg = read_c0_conf(); write_c0_conf(cfg | R30XX_CONF_HALT); - local_irq_enable(); -} - -static void r39xx_wait(void) -{ - if (!need_resched()) - write_c0_conf(read_c0_conf() | TX39_CONF_HALT); - local_irq_enable(); -} - -void r4k_wait(void) -{ - local_irq_enable(); - __r4k_wait(); } /* @@ -58,30 +42,26 @@ void r4k_wait(void) * interrupt is requested" restriction in the MIPS32/MIPS64 architecture makes * using this version a gamble. */ -void r4k_wait_irqoff(void) +void __cpuidle r4k_wait_irqoff(void) { if (!need_resched()) __asm__( " .set push \n" - " .set mips3 \n" + " .set arch=r4000 \n" " wait \n" " .set pop \n"); - local_irq_enable(); - __asm__( - " .globl __pastwait \n" - "__pastwait: \n"); } /* * The RM7000 variant has to handle erratum 38. The workaround is to not * have any pending stores when the WAIT instruction is executed. */ -static void rm7k_wait_irqoff(void) +static void __cpuidle rm7k_wait_irqoff(void) { if (!need_resched()) __asm__( " .set push \n" - " .set mips3 \n" + " .set arch=r4000 \n" " .set noat \n" " mfc0 $1, $12 \n" " sync \n" @@ -89,7 +69,6 @@ static void rm7k_wait_irqoff(void) " wait \n" " mtc0 $1, $12 # stalls until W stage \n" " .set pop \n"); - local_irq_enable(); } /* @@ -97,12 +76,13 @@ static void rm7k_wait_irqoff(void) * since coreclock (and the cp0 counter) stops upon executing it. Only an * interrupt can wake it, so they must be enabled before entering idle modes. */ -static void au1k_wait(void) +static void __cpuidle au1k_wait(void) { unsigned long c0status = read_c0_status() | 1; /* irqs on */ __asm__( - " .set mips3 \n" + " .set push \n" + " .set arch=r4000 \n" " cache 0x14, 0(%0) \n" " cache 0x14, 32(%0) \n" " sync \n" @@ -112,8 +92,10 @@ static void au1k_wait(void) " nop \n" " nop \n" " nop \n" - " .set mips0 \n" + " .set pop \n" : : "r" (au1k_wait), "r" (c0status)); + + raw_local_irq_disable(); } static int __initdata nowait; @@ -136,14 +118,21 @@ void __init check_wait(void) return; } - switch (c->cputype) { + /* + * MIPSr6 specifies that masked interrupts should unblock an executing + * wait instruction, and thus that it is safe for us to use + * r4k_wait_irqoff. Yippee! + */ + if (cpu_has_mips_r6) { + cpu_wait = r4k_wait_irqoff; + return; + } + + switch (current_cpu_type()) { case CPU_R3081: case CPU_R3081E: cpu_wait = r3081_wait; break; - case CPU_TX3927: - cpu_wait = r39xx_wait; - break; case CPU_R4200: /* case CPU_R4300: */ case CPU_R4600: @@ -157,31 +146,54 @@ void __init check_wait(void) case CPU_4KEC: case CPU_4KSC: case CPU_5KC: + case CPU_5KE: case CPU_25KF: case CPU_PR4450: case CPU_BMIPS3300: case CPU_BMIPS4350: case CPU_BMIPS4380: - case CPU_BMIPS5000: case CPU_CAVIUM_OCTEON: case CPU_CAVIUM_OCTEON_PLUS: case CPU_CAVIUM_OCTEON2: - case CPU_JZRISC: - case CPU_LOONGSON1: - case CPU_XLR: - case CPU_XLP: + case CPU_CAVIUM_OCTEON3: + case CPU_XBURST: + case CPU_LOONGSON32: cpu_wait = r4k_wait; break; + case CPU_LOONGSON64: + if ((c->processor_id & (PRID_IMP_MASK | PRID_REV_MASK)) >= + (PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0) || + (c->processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64R) + cpu_wait = r4k_wait; + break; + case CPU_BMIPS5000: + cpu_wait = r4k_wait_irqoff; + break; case CPU_RM7000: cpu_wait = rm7k_wait_irqoff; break; + case CPU_PROAPTIV: + case CPU_P5600: + /* + * Incoming Fast Debug Channel (FDC) data during a wait + * instruction causes the wait never to resume, even if an + * interrupt is received. Avoid using wait at all if FDC data is + * likely to be received. + */ + if (IS_ENABLED(CONFIG_MIPS_EJTAG_FDC_TTY)) + break; + fallthrough; case CPU_M14KC: case CPU_M14KEC: case CPU_24K: case CPU_34K: case CPU_1004K: + case CPU_1074K: + case CPU_INTERAPTIV: + case CPU_M5150: + case CPU_QEMU_GENERIC: cpu_wait = r4k_wait; if (read_c0_config7() & MIPS_CONF7_WII) cpu_wait = r4k_wait_irqoff; @@ -209,7 +221,7 @@ void __init check_wait(void) break; /* - * Another rev is incremeting c0_count at a reduced clock + * Another rev is incrementing c0_count at a reduced clock * rate while in WAIT mode. So we basically have the choice * between using the cp0 timer as clocksource or avoiding * the WAIT instruction. Until more details are known, @@ -217,29 +229,24 @@ void __init check_wait(void) cpu_wait = r4k_wait; */ break; - case CPU_RM9000: - if ((c->processor_id & 0x00ff) >= 0x40) - cpu_wait = r4k_wait; - break; default: break; } } -static void smtc_idle_hook(void) +__cpuidle void arch_cpu_idle(void) { -#ifdef CONFIG_MIPS_MT_SMTC - void smtc_idle_loop_hook(void); - - smtc_idle_loop_hook(); -#endif + if (cpu_wait) + cpu_wait(); } -void arch_cpu_idle(void) +#ifdef CONFIG_CPU_IDLE + +__cpuidle int mips_cpuidle_wait_enter(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int index) { - smtc_idle_hook(); - if (cpu_wait) - cpu_wait(); - else - local_irq_enable(); + arch_cpu_idle(); + return index; } + +#endif diff --git a/arch/mips/kernel/irq-gic.c b/arch/mips/kernel/irq-gic.c deleted file mode 100644 index 5b5ddb231f26..000000000000 --- a/arch/mips/kernel/irq-gic.c +++ /dev/null @@ -1,366 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2008 Ralf Baechle (ralf@linux-mips.org) - * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. - */ -#include <linux/bitmap.h> -#include <linux/init.h> -#include <linux/smp.h> -#include <linux/irq.h> -#include <linux/clocksource.h> - -#include <asm/io.h> -#include <asm/gic.h> -#include <asm/setup.h> -#include <asm/traps.h> -#include <asm/gcmpregs.h> -#include <linux/hardirq.h> -#include <asm-generic/bitops/find.h> - -unsigned int gic_frequency; -unsigned int gic_present; -unsigned long _gic_base; -unsigned int gic_irq_base; -unsigned int gic_irq_flags[GIC_NUM_INTRS]; - -/* The index into this array is the vector # of the interrupt. */ -struct gic_shared_intr_map gic_shared_intr_map[GIC_NUM_INTRS]; - -static struct gic_pcpu_mask pcpu_masks[NR_CPUS]; -static struct gic_pending_regs pending_regs[NR_CPUS]; -static struct gic_intrmask_regs intrmask_regs[NR_CPUS]; - -#if defined(CONFIG_CSRC_GIC) || defined(CONFIG_CEVT_GIC) -cycle_t gic_read_count(void) -{ - unsigned int hi, hi2, lo; - - do { - GICREAD(GIC_REG(SHARED, GIC_SH_COUNTER_63_32), hi); - GICREAD(GIC_REG(SHARED, GIC_SH_COUNTER_31_00), lo); - GICREAD(GIC_REG(SHARED, GIC_SH_COUNTER_63_32), hi2); - } while (hi2 != hi); - - return (((cycle_t) hi) << 32) + lo; -} - -void gic_write_compare(cycle_t cnt) -{ - GICWRITE(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI), - (int)(cnt >> 32)); - GICWRITE(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO), - (int)(cnt & 0xffffffff)); -} - -cycle_t gic_read_compare(void) -{ - unsigned int hi, lo; - - GICREAD(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI), hi); - GICREAD(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO), lo); - - return (((cycle_t) hi) << 32) + lo; -} -#endif - -unsigned int gic_get_timer_pending(void) -{ - unsigned int vpe_pending; - - GICWRITE(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), 0); - GICREAD(GIC_REG(VPE_OTHER, GIC_VPE_PEND), vpe_pending); - return (vpe_pending & GIC_VPE_PEND_TIMER_MSK); -} - -void gic_bind_eic_interrupt(int irq, int set) -{ - /* Convert irq vector # to hw int # */ - irq -= GIC_PIN_TO_VEC_OFFSET; - - /* Set irq to use shadow set */ - GICWRITE(GIC_REG_ADDR(VPE_LOCAL, GIC_VPE_EIC_SS(irq)), set); -} - -void gic_send_ipi(unsigned int intr) -{ - GICWRITE(GIC_REG(SHARED, GIC_SH_WEDGE), 0x80000000 | intr); -} - -static void gic_eic_irq_dispatch(void) -{ - unsigned int cause = read_c0_cause(); - int irq; - - irq = (cause & ST0_IM) >> STATUSB_IP2; - if (irq == 0) - irq = -1; - - if (irq >= 0) - do_IRQ(gic_irq_base + irq); - else - spurious_interrupt(); -} - -static void __init vpe_local_setup(unsigned int numvpes) -{ - unsigned long timer_intr = GIC_INT_TMR; - unsigned long perf_intr = GIC_INT_PERFCTR; - unsigned int vpe_ctl; - int i; - - if (cpu_has_veic) { - /* - * GIC timer interrupt -> CPU HW Int X (vector X+2) -> - * map to pin X+2-1 (since GIC adds 1) - */ - timer_intr += (GIC_CPU_TO_VEC_OFFSET - GIC_PIN_TO_VEC_OFFSET); - /* - * GIC perfcnt interrupt -> CPU HW Int X (vector X+2) -> - * map to pin X+2-1 (since GIC adds 1) - */ - perf_intr += (GIC_CPU_TO_VEC_OFFSET - GIC_PIN_TO_VEC_OFFSET); - } - - /* - * Setup the default performance counter timer interrupts - * for all VPEs - */ - for (i = 0; i < numvpes; i++) { - GICWRITE(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), i); - - /* Are Interrupts locally routable? */ - GICREAD(GIC_REG(VPE_OTHER, GIC_VPE_CTL), vpe_ctl); - if (vpe_ctl & GIC_VPE_CTL_TIMER_RTBL_MSK) - GICWRITE(GIC_REG(VPE_OTHER, GIC_VPE_TIMER_MAP), - GIC_MAP_TO_PIN_MSK | timer_intr); - if (cpu_has_veic) { - set_vi_handler(timer_intr + GIC_PIN_TO_VEC_OFFSET, - gic_eic_irq_dispatch); - gic_shared_intr_map[timer_intr + GIC_PIN_TO_VEC_OFFSET].local_intr_mask |= GIC_VPE_RMASK_TIMER_MSK; - } - - if (vpe_ctl & GIC_VPE_CTL_PERFCNT_RTBL_MSK) - GICWRITE(GIC_REG(VPE_OTHER, GIC_VPE_PERFCTR_MAP), - GIC_MAP_TO_PIN_MSK | perf_intr); - if (cpu_has_veic) { - set_vi_handler(perf_intr + GIC_PIN_TO_VEC_OFFSET, gic_eic_irq_dispatch); - gic_shared_intr_map[perf_intr + GIC_PIN_TO_VEC_OFFSET].local_intr_mask |= GIC_VPE_RMASK_PERFCNT_MSK; - } - } -} - -unsigned int gic_compare_int(void) -{ - unsigned int pending; - - GICREAD(GIC_REG(VPE_LOCAL, GIC_VPE_PEND), pending); - if (pending & GIC_VPE_PEND_CMP_MSK) - return 1; - else - return 0; -} - -unsigned int gic_get_int(void) -{ - unsigned int i; - unsigned long *pending, *intrmask, *pcpu_mask; - unsigned long *pending_abs, *intrmask_abs; - - /* Get per-cpu bitmaps */ - pending = pending_regs[smp_processor_id()].pending; - intrmask = intrmask_regs[smp_processor_id()].intrmask; - pcpu_mask = pcpu_masks[smp_processor_id()].pcpu_mask; - - pending_abs = (unsigned long *) GIC_REG_ABS_ADDR(SHARED, - GIC_SH_PEND_31_0_OFS); - intrmask_abs = (unsigned long *) GIC_REG_ABS_ADDR(SHARED, - GIC_SH_MASK_31_0_OFS); - - for (i = 0; i < BITS_TO_LONGS(GIC_NUM_INTRS); i++) { - GICREAD(*pending_abs, pending[i]); - GICREAD(*intrmask_abs, intrmask[i]); - pending_abs++; - intrmask_abs++; - } - - bitmap_and(pending, pending, intrmask, GIC_NUM_INTRS); - bitmap_and(pending, pending, pcpu_mask, GIC_NUM_INTRS); - - return find_first_bit(pending, GIC_NUM_INTRS); -} - -static void gic_mask_irq(struct irq_data *d) -{ - GIC_CLR_INTR_MASK(d->irq - gic_irq_base); -} - -static void gic_unmask_irq(struct irq_data *d) -{ - GIC_SET_INTR_MASK(d->irq - gic_irq_base); -} - -#ifdef CONFIG_SMP -static DEFINE_SPINLOCK(gic_lock); - -static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask, - bool force) -{ - unsigned int irq = (d->irq - gic_irq_base); - cpumask_t tmp = CPU_MASK_NONE; - unsigned long flags; - int i; - - cpumask_and(&tmp, cpumask, cpu_online_mask); - if (cpus_empty(tmp)) - return -1; - - /* Assumption : cpumask refers to a single CPU */ - spin_lock_irqsave(&gic_lock, flags); - - /* Re-route this IRQ */ - GIC_SH_MAP_TO_VPE_SMASK(irq, first_cpu(tmp)); - - /* Update the pcpu_masks */ - for (i = 0; i < NR_CPUS; i++) - clear_bit(irq, pcpu_masks[i].pcpu_mask); - set_bit(irq, pcpu_masks[first_cpu(tmp)].pcpu_mask); - - cpumask_copy(d->affinity, cpumask); - spin_unlock_irqrestore(&gic_lock, flags); - - return IRQ_SET_MASK_OK_NOCOPY; -} -#endif - -static struct irq_chip gic_irq_controller = { - .name = "MIPS GIC", - .irq_ack = gic_irq_ack, - .irq_mask = gic_mask_irq, - .irq_mask_ack = gic_mask_irq, - .irq_unmask = gic_unmask_irq, - .irq_eoi = gic_finish_irq, -#ifdef CONFIG_SMP - .irq_set_affinity = gic_set_affinity, -#endif -}; - -static void __init gic_setup_intr(unsigned int intr, unsigned int cpu, - unsigned int pin, unsigned int polarity, unsigned int trigtype, - unsigned int flags) -{ - struct gic_shared_intr_map *map_ptr; - - /* Setup Intr to Pin mapping */ - if (pin & GIC_MAP_TO_NMI_MSK) { - GICWRITE(GIC_REG_ADDR(SHARED, GIC_SH_MAP_TO_PIN(intr)), pin); - /* FIXME: hack to route NMI to all cpu's */ - for (cpu = 0; cpu < NR_CPUS; cpu += 32) { - GICWRITE(GIC_REG_ADDR(SHARED, - GIC_SH_MAP_TO_VPE_REG_OFF(intr, cpu)), - 0xffffffff); - } - } else { - GICWRITE(GIC_REG_ADDR(SHARED, GIC_SH_MAP_TO_PIN(intr)), - GIC_MAP_TO_PIN_MSK | pin); - /* Setup Intr to CPU mapping */ - GIC_SH_MAP_TO_VPE_SMASK(intr, cpu); - if (cpu_has_veic) { - set_vi_handler(pin + GIC_PIN_TO_VEC_OFFSET, - gic_eic_irq_dispatch); - map_ptr = &gic_shared_intr_map[pin + GIC_PIN_TO_VEC_OFFSET]; - if (map_ptr->num_shared_intr >= GIC_MAX_SHARED_INTR) - BUG(); - map_ptr->intr_list[map_ptr->num_shared_intr++] = intr; - } - } - - /* Setup Intr Polarity */ - GIC_SET_POLARITY(intr, polarity); - - /* Setup Intr Trigger Type */ - GIC_SET_TRIGGER(intr, trigtype); - - /* Init Intr Masks */ - GIC_CLR_INTR_MASK(intr); - /* Initialise per-cpu Interrupt software masks */ - if (flags & GIC_FLAG_IPI) - set_bit(intr, pcpu_masks[cpu].pcpu_mask); - if ((flags & GIC_FLAG_TRANSPARENT) && (cpu_has_veic == 0)) - GIC_SET_INTR_MASK(intr); - if (trigtype == GIC_TRIG_EDGE) - gic_irq_flags[intr] |= GIC_TRIG_EDGE; -} - -static void __init gic_basic_init(int numintrs, int numvpes, - struct gic_intr_map *intrmap, int mapsize) -{ - unsigned int i, cpu; - unsigned int pin_offset = 0; - - board_bind_eic_interrupt = &gic_bind_eic_interrupt; - - /* Setup defaults */ - for (i = 0; i < numintrs; i++) { - GIC_SET_POLARITY(i, GIC_POL_POS); - GIC_SET_TRIGGER(i, GIC_TRIG_LEVEL); - GIC_CLR_INTR_MASK(i); - if (i < GIC_NUM_INTRS) { - gic_irq_flags[i] = 0; - gic_shared_intr_map[i].num_shared_intr = 0; - gic_shared_intr_map[i].local_intr_mask = 0; - } - } - - /* - * In EIC mode, the HW_INT# is offset by (2-1). Need to subtract - * one because the GIC will add one (since 0=no intr). - */ - if (cpu_has_veic) - pin_offset = (GIC_CPU_TO_VEC_OFFSET - GIC_PIN_TO_VEC_OFFSET); - - /* Setup specifics */ - for (i = 0; i < mapsize; i++) { - cpu = intrmap[i].cpunum; - if (cpu == GIC_UNUSED) - continue; - if (cpu == 0 && i != 0 && intrmap[i].flags == 0) - continue; - gic_setup_intr(i, - intrmap[i].cpunum, - intrmap[i].pin + pin_offset, - intrmap[i].polarity, - intrmap[i].trigtype, - intrmap[i].flags); - } - - vpe_local_setup(numvpes); -} - -void __init gic_init(unsigned long gic_base_addr, - unsigned long gic_addrspace_size, - struct gic_intr_map *intr_map, unsigned int intr_map_size, - unsigned int irqbase) -{ - unsigned int gicconfig; - int numvpes, numintrs; - - _gic_base = (unsigned long) ioremap_nocache(gic_base_addr, - gic_addrspace_size); - gic_irq_base = irqbase; - - GICREAD(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig); - numintrs = (gicconfig & GIC_SH_CONFIG_NUMINTRS_MSK) >> - GIC_SH_CONFIG_NUMINTRS_SHF; - numintrs = ((numintrs + 1) * 8); - - numvpes = (gicconfig & GIC_SH_CONFIG_NUMVPES_MSK) >> - GIC_SH_CONFIG_NUMVPES_SHF; - numvpes = numvpes + 1; - - gic_basic_init(numintrs, numvpes, intr_map, intr_map_size); - - gic_platform_init(numintrs, &gic_irq_controller); -} diff --git a/arch/mips/kernel/irq-gt641xx.c b/arch/mips/kernel/irq-gt641xx.c index 44a1f792e399..93bcf5736a6f 100644 --- a/arch/mips/kernel/irq-gt641xx.c +++ b/arch/mips/kernel/irq-gt641xx.c @@ -1,21 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * GT641xx IRQ routines. * * Copyright (C) 2007 Yoichi Yuasa <yuasa@linux-mips.org> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include <linux/hardirq.h> #include <linux/init.h> diff --git a/arch/mips/kernel/irq-msc01.c b/arch/mips/kernel/irq-msc01.c index fab40f7d2e03..ab511b64a19e 100644 --- a/arch/mips/kernel/irq-msc01.c +++ b/arch/mips/kernel/irq-msc01.c @@ -1,8 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. * * Copyright (c) 2004 MIPS Inc * Author: chris@mips.com @@ -53,13 +50,9 @@ static inline void unmask_msc_irq(struct irq_data *d) */ static void level_mask_and_ack_msc_irq(struct irq_data *d) { - unsigned int irq = d->irq; - mask_msc_irq(d); if (!cpu_has_veic) MSCIC_WRITE(MSC01_IC_EOI, 0); - /* This actually needs to be a call into platform code */ - smtc_im_ack_irq(irq); } /* @@ -78,7 +71,6 @@ static void edge_mask_and_ack_msc_irq(struct irq_data *d) MSCIC_WRITE(MSC01_IC_SUP+irq*8, r | ~MSC01_IC_SUP_EDGE_BIT); MSCIC_WRITE(MSC01_IC_SUP+irq*8, r); } - smtc_im_ack_irq(irq); } /* @@ -131,7 +123,7 @@ void __init init_msc_irqs(unsigned long icubase, unsigned int irqbase, msc_irqma board_bind_eic_interrupt = &msc_bind_eic_interrupt; - for (; nirq >= 0; nirq--, imp++) { + for (; nirq > 0; nirq--, imp++) { int n = imp->im_irq; switch (imp->im_type) { diff --git a/arch/mips/kernel/irq-rm7000.c b/arch/mips/kernel/irq-rm7000.c deleted file mode 100644 index 26f4e4c9db1a..000000000000 --- a/arch/mips/kernel/irq-rm7000.c +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2003 Ralf Baechle - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * Handler for RM7000 extended interrupts. These are a non-standard - * feature so we handle them separately from standard interrupts. - */ -#include <linux/init.h> -#include <linux/interrupt.h> -#include <linux/irq.h> -#include <linux/kernel.h> - -#include <asm/irq_cpu.h> -#include <asm/mipsregs.h> - -static inline void unmask_rm7k_irq(struct irq_data *d) -{ - set_c0_intcontrol(0x100 << (d->irq - RM7K_CPU_IRQ_BASE)); -} - -static inline void mask_rm7k_irq(struct irq_data *d) -{ - clear_c0_intcontrol(0x100 << (d->irq - RM7K_CPU_IRQ_BASE)); -} - -static struct irq_chip rm7k_irq_controller = { - .name = "RM7000", - .irq_ack = mask_rm7k_irq, - .irq_mask = mask_rm7k_irq, - .irq_mask_ack = mask_rm7k_irq, - .irq_unmask = unmask_rm7k_irq, - .irq_eoi = unmask_rm7k_irq -}; - -void __init rm7k_cpu_irq_init(void) -{ - int base = RM7K_CPU_IRQ_BASE; - int i; - - clear_c0_intcontrol(0x00000f00); /* Mask all */ - - for (i = base; i < base + 4; i++) - irq_set_chip_and_handler(i, &rm7k_irq_controller, - handle_percpu_irq); -} diff --git a/arch/mips/kernel/irq.c b/arch/mips/kernel/irq.c index d1fea7a054be..5e11582fe308 100644 --- a/arch/mips/kernel/irq.c +++ b/arch/mips/kernel/irq.c @@ -21,51 +21,12 @@ #include <linux/kallsyms.h> #include <linux/kgdb.h> #include <linux/ftrace.h> +#include <linux/irqdomain.h> #include <linux/atomic.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> -#ifdef CONFIG_KGDB -int kgdb_early_setup; -#endif - -static unsigned long irq_map[NR_IRQS / BITS_PER_LONG]; - -int allocate_irqno(void) -{ - int irq; - -again: - irq = find_first_zero_bit(irq_map, NR_IRQS); - - if (irq >= NR_IRQS) - return -ENOSPC; - - if (test_and_set_bit(irq, irq_map)) - goto again; - - return irq; -} - -/* - * Allocate the 16 legacy interrupts for i8259 devices. This happens early - * in the kernel initialization so treating allocation failure as BUG() is - * ok. - */ -void __init alloc_legacy_irqno(void) -{ - int i; - - for (i = 0; i <= 16; i++) - BUG_ON(test_and_set_bit(i, irq_map)); -} - -void free_irqno(unsigned int irq) -{ - smp_mb__before_clear_bit(); - clear_bit(irq, irq_map); - smp_mb__after_clear_bit(); -} +void *irq_stack[NR_CPUS]; /* * 'what should we do if we get a hw irq event on an illegal vector'. @@ -73,7 +34,6 @@ void free_irqno(unsigned int irq) */ void ack_bad_irq(unsigned int irq) { - smtc_im_ack_irq(irq); printk("unexpected IRQ # %d\n", irq); } @@ -93,24 +53,26 @@ asmlinkage void spurious_interrupt(void) void __init init_IRQ(void) { int i; - -#ifdef CONFIG_KGDB - if (kgdb_early_setup) - return; -#endif + unsigned int order = get_order(IRQ_STACK_SIZE); for (i = 0; i < NR_IRQS; i++) irq_set_noprobe(i); + if (cpu_has_veic) + clear_c0_status(ST0_IM); + arch_init_irq(); -#ifdef CONFIG_KGDB - if (!kgdb_early_setup) - kgdb_early_setup = 1; -#endif + for_each_possible_cpu(i) { + void *s = (void *)__get_free_pages(GFP_KERNEL, order); + + irq_stack[i] = s; + pr_debug("CPU%d IRQ stack at 0x%p - 0x%p\n", i, + irq_stack[i], irq_stack[i] + IRQ_STACK_SIZE); + } } -#ifdef DEBUG_STACKOVERFLOW +#ifdef CONFIG_DEBUG_STACKOVERFLOW static inline void check_stack_overflow(void) { unsigned long sp; @@ -142,23 +104,16 @@ void __irq_entry do_IRQ(unsigned int irq) { irq_enter(); check_stack_overflow(); - if (!smtc_handle_on_other_cpu(irq)) - generic_handle_irq(irq); + generic_handle_irq(irq); irq_exit(); } -#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF -/* - * To avoid inefficient and in some cases pathological re-checking of - * IRQ affinity, we have this variant that skips the affinity check. - */ - -void __irq_entry do_IRQ_no_affinity(unsigned int irq) +#ifdef CONFIG_IRQ_DOMAIN +void __irq_entry do_domain_IRQ(struct irq_domain *domain, unsigned int hwirq) { irq_enter(); - smtc_im_backstop(irq); - generic_handle_irq(irq); + check_stack_overflow(); + generic_handle_domain_irq(domain, hwirq); irq_exit(); } - -#endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */ +#endif diff --git a/arch/mips/kernel/irq_cpu.c b/arch/mips/kernel/irq_cpu.c deleted file mode 100644 index 72ef2d25cbf2..000000000000 --- a/arch/mips/kernel/irq_cpu.c +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2001 MontaVista Software Inc. - * Author: Jun Sun, jsun@mvista.com or jsun@junsun.net - * - * Copyright (C) 2001 Ralf Baechle - * Copyright (C) 2005 MIPS Technologies, Inc. All rights reserved. - * Author: Maciej W. Rozycki <macro@mips.com> - * - * This file define the irq handler for MIPS CPU interrupts. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ - -/* - * Almost all MIPS CPUs define 8 interrupt sources. They are typically - * level triggered (i.e., cannot be cleared from CPU; must be cleared from - * device). The first two are software interrupts which we don't really - * use or support. The last one is usually the CPU timer interrupt if - * counter register is present or, for CPUs with an external FPU, by - * convention it's the FPU exception interrupt. - * - * Don't even think about using this on SMP. You have been warned. - * - * This file exports one global function: - * void mips_cpu_irq_init(void); - */ -#include <linux/init.h> -#include <linux/interrupt.h> -#include <linux/kernel.h> -#include <linux/irq.h> -#include <linux/irqdomain.h> - -#include <asm/irq_cpu.h> -#include <asm/mipsregs.h> -#include <asm/mipsmtregs.h> - -static inline void unmask_mips_irq(struct irq_data *d) -{ - set_c0_status(0x100 << (d->irq - MIPS_CPU_IRQ_BASE)); - irq_enable_hazard(); -} - -static inline void mask_mips_irq(struct irq_data *d) -{ - clear_c0_status(0x100 << (d->irq - MIPS_CPU_IRQ_BASE)); - irq_disable_hazard(); -} - -static struct irq_chip mips_cpu_irq_controller = { - .name = "MIPS", - .irq_ack = mask_mips_irq, - .irq_mask = mask_mips_irq, - .irq_mask_ack = mask_mips_irq, - .irq_unmask = unmask_mips_irq, - .irq_eoi = unmask_mips_irq, -}; - -/* - * Basically the same as above but taking care of all the MT stuff - */ - -static unsigned int mips_mt_cpu_irq_startup(struct irq_data *d) -{ - unsigned int vpflags = dvpe(); - - clear_c0_cause(0x100 << (d->irq - MIPS_CPU_IRQ_BASE)); - evpe(vpflags); - unmask_mips_irq(d); - return 0; -} - -/* - * While we ack the interrupt interrupts are disabled and thus we don't need - * to deal with concurrency issues. Same for mips_cpu_irq_end. - */ -static void mips_mt_cpu_irq_ack(struct irq_data *d) -{ - unsigned int vpflags = dvpe(); - clear_c0_cause(0x100 << (d->irq - MIPS_CPU_IRQ_BASE)); - evpe(vpflags); - mask_mips_irq(d); -} - -static struct irq_chip mips_mt_cpu_irq_controller = { - .name = "MIPS", - .irq_startup = mips_mt_cpu_irq_startup, - .irq_ack = mips_mt_cpu_irq_ack, - .irq_mask = mask_mips_irq, - .irq_mask_ack = mips_mt_cpu_irq_ack, - .irq_unmask = unmask_mips_irq, - .irq_eoi = unmask_mips_irq, -}; - -void __init mips_cpu_irq_init(void) -{ - int irq_base = MIPS_CPU_IRQ_BASE; - int i; - - /* Mask interrupts. */ - clear_c0_status(ST0_IM); - clear_c0_cause(CAUSEF_IP); - - /* Software interrupts are used for MT/CMT IPI */ - for (i = irq_base; i < irq_base + 2; i++) - irq_set_chip_and_handler(i, cpu_has_mipsmt ? - &mips_mt_cpu_irq_controller : - &mips_cpu_irq_controller, - handle_percpu_irq); - - for (i = irq_base + 2; i < irq_base + 8; i++) - irq_set_chip_and_handler(i, &mips_cpu_irq_controller, - handle_percpu_irq); -} - -#ifdef CONFIG_IRQ_DOMAIN -static int mips_cpu_intc_map(struct irq_domain *d, unsigned int irq, - irq_hw_number_t hw) -{ - static struct irq_chip *chip; - - if (hw < 2 && cpu_has_mipsmt) { - /* Software interrupts are used for MT/CMT IPI */ - chip = &mips_mt_cpu_irq_controller; - } else { - chip = &mips_cpu_irq_controller; - } - - irq_set_chip_and_handler(irq, chip, handle_percpu_irq); - - return 0; -} - -static const struct irq_domain_ops mips_cpu_intc_irq_domain_ops = { - .map = mips_cpu_intc_map, - .xlate = irq_domain_xlate_onecell, -}; - -int __init mips_cpu_intc_init(struct device_node *of_node, - struct device_node *parent) -{ - struct irq_domain *domain; - - /* Mask interrupts. */ - clear_c0_status(ST0_IM); - clear_c0_cause(CAUSEF_IP); - - domain = irq_domain_add_legacy(of_node, 8, MIPS_CPU_IRQ_BASE, 0, - &mips_cpu_intc_irq_domain_ops, NULL); - if (!domain) - panic("Failed to add irqdomain for MIPS CPU\n"); - - return 0; -} -#endif /* CONFIG_IRQ_DOMAIN */ diff --git a/arch/mips/kernel/irq_txx9.c b/arch/mips/kernel/irq_txx9.c index ab00e490482f..af3ef4c9f7de 100644 --- a/arch/mips/kernel/irq_txx9.c +++ b/arch/mips/kernel/irq_txx9.c @@ -72,11 +72,6 @@ static void txx9_irq_unmask(struct irq_data *d) __raw_writel((__raw_readl(ilrp) & ~(0xff << ofs)) | (txx9irq[irq_nr].level << ofs), ilrp); -#ifdef CONFIG_CPU_TX39XX - /* update IRCSR */ - __raw_writel(0, &txx9_ircptr->imr); - __raw_writel(irc_elevel, &txx9_ircptr->imr); -#endif } static inline void txx9_irq_mask(struct irq_data *d) @@ -88,15 +83,7 @@ static inline void txx9_irq_mask(struct irq_data *d) __raw_writel((__raw_readl(ilrp) & ~(0xff << ofs)) | (irc_dlevel << ofs), ilrp); -#ifdef CONFIG_CPU_TX39XX - /* update IRCSR */ - __raw_writel(0, &txx9_ircptr->imr); - __raw_writel(irc_elevel, &txx9_ircptr->imr); - /* flush write buffer */ - __raw_readl(&txx9_ircptr->ssr); -#else mmiowb(); -#endif } static void txx9_irq_mask_ack(struct irq_data *d) diff --git a/arch/mips/kernel/jump_label.c b/arch/mips/kernel/jump_label.c index 6001610cfe55..f7978d50a2ba 100644 --- a/arch/mips/kernel/jump_label.c +++ b/arch/mips/kernel/jump_label.c @@ -16,39 +16,94 @@ #include <asm/cacheflush.h> #include <asm/inst.h> -#ifdef HAVE_JUMP_LABEL - -#define J_RANGE_MASK ((1ul << 28) - 1) +/* + * Define parameters for the standard MIPS and the microMIPS jump + * instruction encoding respectively: + * + * - the ISA bit of the target, either 0 or 1 respectively, + * + * - the amount the jump target address is shifted right to fit in the + * immediate field of the machine instruction, either 2 or 1, + * + * - the mask determining the size of the jump region relative to the + * delay-slot instruction, either 256MB or 128MB, + * + * - the jump target alignment, either 4 or 2 bytes. + */ +#define J_ISA_BIT IS_ENABLED(CONFIG_CPU_MICROMIPS) +#define J_RANGE_SHIFT (2 - J_ISA_BIT) +#define J_RANGE_MASK ((1ul << (26 + J_RANGE_SHIFT)) - 1) +#define J_ALIGN_MASK ((1ul << J_RANGE_SHIFT) - 1) void arch_jump_label_transform(struct jump_entry *e, enum jump_label_type type) { + union mips_instruction *insn_p; union mips_instruction insn; - union mips_instruction *insn_p = - (union mips_instruction *)(unsigned long)e->code; + long offset; + + insn_p = (union mips_instruction *)msk_isa16_mode(e->code); - /* Jump only works within a 256MB aligned region. */ - BUG_ON((e->target & ~J_RANGE_MASK) != (e->code & ~J_RANGE_MASK)); + /* Target must have the right alignment and ISA must be preserved. */ + BUG_ON((e->target & J_ALIGN_MASK) != J_ISA_BIT); - /* Target must have 4 byte alignment. */ - BUG_ON((e->target & 3) != 0); + if (type == JUMP_LABEL_JMP) { + if (!IS_ENABLED(CONFIG_CPU_MICROMIPS) && MIPS_ISA_REV >= 6) { + offset = e->target - ((unsigned long)insn_p + 4); + offset >>= 2; - if (type == JUMP_LABEL_ENABLE) { - insn.j_format.opcode = j_op; - insn.j_format.target = (e->target & J_RANGE_MASK) >> 2; + /* + * The branch offset must fit in the instruction's 26 + * bit field. + */ + WARN_ON((offset >= (long)BIT(25)) || + (offset < -(long)BIT(25))); + + insn.j_format.opcode = bc6_op; + insn.j_format.target = offset; + } else { + /* + * Jump only works within an aligned region its delay + * slot is in. + */ + WARN_ON((e->target & ~J_RANGE_MASK) != + ((e->code + 4) & ~J_RANGE_MASK)); + + insn.j_format.opcode = J_ISA_BIT ? mm_j32_op : j_op; + insn.j_format.target = e->target >> J_RANGE_SHIFT; + } } else { insn.word = 0; /* nop */ } - get_online_cpus(); mutex_lock(&text_mutex); - *insn_p = insn; + if (IS_ENABLED(CONFIG_CPU_MICROMIPS)) { + insn_p->halfword[0] = insn.word >> 16; + insn_p->halfword[1] = insn.word; + } else + *insn_p = insn; flush_icache_range((unsigned long)insn_p, (unsigned long)insn_p + sizeof(*insn_p)); mutex_unlock(&text_mutex); - put_online_cpus(); } -#endif /* HAVE_JUMP_LABEL */ +#ifdef CONFIG_MODULES +void jump_label_apply_nops(struct module *mod) +{ + struct jump_entry *iter_start = mod->jump_entries; + struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; + struct jump_entry *iter; + + /* if the module doesn't have jump label entries, just return */ + if (iter_start == iter_stop) + return; + + for (iter = iter_start; iter < iter_stop; iter++) { + /* Only write NOPs for arch_branch_static(). */ + if (jump_label_init_type(iter) == JUMP_LABEL_NOP) + arch_jump_label_transform(iter, JUMP_LABEL_NOP); + } +} +#endif diff --git a/arch/mips/kernel/kgdb.c b/arch/mips/kernel/kgdb.c index fcaac2f132f0..09a2d7bb9eef 100644 --- a/arch/mips/kernel/kgdb.c +++ b/arch/mips/kernel/kgdb.c @@ -32,6 +32,7 @@ #include <asm/cacheflush.h> #include <asm/processor.h> #include <asm/sigcontext.h> +#include <asm/irq_regs.h> static struct hard_trap_info { unsigned char tt; /* Trap type code for MIPS R3xxx and R4xxx */ @@ -206,18 +207,6 @@ void arch_kgdb_breakpoint(void) ".set\treorder"); } -static void kgdb_call_nmi_hook(void *ignored) -{ - kgdb_nmicallback(raw_smp_processor_id(), NULL); -} - -void kgdb_roundup_cpus(unsigned long flags) -{ - local_irq_enable(); - smp_call_function(kgdb_call_nmi_hook, NULL, 0); - local_irq_disable(); -} - static int compute_signal(int tt) { struct hard_trap_info *ht; @@ -236,9 +225,6 @@ static int compute_signal(int tt) void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) { int reg; - struct thread_info *ti = task_thread_info(p); - unsigned long ksp = (unsigned long)ti + THREAD_SIZE - 32; - struct pt_regs *regs = (struct pt_regs *)ksp - 1; #if (KGDB_GDB_REG_SIZE == 32) u32 *ptr = (u32 *)gdb_regs; #else @@ -246,25 +232,46 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) #endif for (reg = 0; reg < 16; reg++) - *(ptr++) = regs->regs[reg]; + *(ptr++) = 0; /* S0 - S7 */ - for (reg = 16; reg < 24; reg++) - *(ptr++) = regs->regs[reg]; + *(ptr++) = p->thread.reg16; + *(ptr++) = p->thread.reg17; + *(ptr++) = p->thread.reg18; + *(ptr++) = p->thread.reg19; + *(ptr++) = p->thread.reg20; + *(ptr++) = p->thread.reg21; + *(ptr++) = p->thread.reg22; + *(ptr++) = p->thread.reg23; for (reg = 24; reg < 28; reg++) *(ptr++) = 0; /* GP, SP, FP, RA */ - for (reg = 28; reg < 32; reg++) - *(ptr++) = regs->regs[reg]; - - *(ptr++) = regs->cp0_status; - *(ptr++) = regs->lo; - *(ptr++) = regs->hi; - *(ptr++) = regs->cp0_badvaddr; - *(ptr++) = regs->cp0_cause; - *(ptr++) = regs->cp0_epc; + *(ptr++) = (long)p; + *(ptr++) = p->thread.reg29; + *(ptr++) = p->thread.reg30; + *(ptr++) = p->thread.reg31; + + *(ptr++) = p->thread.cp0_status; + + /* lo, hi */ + *(ptr++) = 0; + *(ptr++) = 0; + + /* + * BadVAddr, Cause + * Ideally these would come from the last exception frame up the stack + * but that requires unwinding, otherwise we can't know much for sure. + */ + *(ptr++) = 0; + *(ptr++) = 0; + + /* + * PC + * use return address (RA), i.e. the moment after return from resume() + */ + *(ptr++) = p->thread.reg31; } void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long pc) @@ -360,22 +367,16 @@ int kgdb_arch_handle_exception(int vector, int signo, int err_code, return -1; } -struct kgdb_arch arch_kgdb_ops; +const struct kgdb_arch arch_kgdb_ops = { +#ifdef CONFIG_CPU_BIG_ENDIAN + .gdb_bpt_instr = { spec_op << 2, 0x00, 0x00, break_op }, +#else + .gdb_bpt_instr = { break_op, 0x00, 0x00, spec_op << 2 }, +#endif +}; -/* - * We use kgdb_early_setup so that functions we need to call now don't - * cause trouble when called again later. - */ int kgdb_arch_init(void) { - union mips_instruction insn = { - .r_format = { - .opcode = spec_op, - .func = break_op, - } - }; - memcpy(arch_kgdb_ops.gdb_bpt_instr, insn.byte, BREAK_INSTR_SIZE); - register_die_notifier(&kgdb_notifier); return 0; diff --git a/arch/mips/kernel/kprobes.c b/arch/mips/kernel/kprobes.c index 1f8187ab0997..dc39f5b3fb83 100644 --- a/arch/mips/kernel/kprobes.c +++ b/arch/mips/kernel/kprobes.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Kernel Probes (KProbes) * arch/mips/kernel/kprobes.c @@ -8,21 +9,10 @@ * Some portions copied from the powerpc version. * * Copyright (C) IBM Corporation, 2002, 2004 - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#define pr_fmt(fmt) "kprobes: " fmt + #include <linux/kprobes.h> #include <linux/preempt.h> #include <linux/uaccess.h> @@ -32,7 +22,8 @@ #include <asm/ptrace.h> #include <asm/branch.h> #include <asm/break.h> -#include <asm/inst.h> + +#include "probes-common.h" static const union mips_instruction breakpoint_insn = { .b_format = { @@ -53,75 +44,20 @@ static const union mips_instruction breakpoint2_insn = { DEFINE_PER_CPU(struct kprobe *, current_kprobe); DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); -static int __kprobes insn_has_delayslot(union mips_instruction insn) +static int insn_has_delayslot(union mips_instruction insn) { - switch (insn.i_format.opcode) { - - /* - * This group contains: - * jr and jalr are in r_format format. - */ - case spec_op: - switch (insn.r_format.func) { - case jr_op: - case jalr_op: - break; - default: - goto insn_ok; - } - - /* - * This group contains: - * bltz_op, bgez_op, bltzl_op, bgezl_op, - * bltzal_op, bgezal_op, bltzall_op, bgezall_op. - */ - case bcond_op: - - /* - * These are unconditional and in j_format. - */ - case jal_op: - case j_op: - - /* - * These are conditional and in i_format. - */ - case beq_op: - case beql_op: - case bne_op: - case bnel_op: - case blez_op: - case blezl_op: - case bgtz_op: - case bgtzl_op: - - /* - * These are the FPA/cp1 branch instructions. - */ - case cop1_op: - -#ifdef CONFIG_CPU_CAVIUM_OCTEON - case lwc2_op: /* This is bbit0 on Octeon */ - case ldc2_op: /* This is bbit032 on Octeon */ - case swc2_op: /* This is bbit1 on Octeon */ - case sdc2_op: /* This is bbit132 on Octeon */ -#endif - return 1; - default: - break; - } -insn_ok: - return 0; + return __insn_has_delay_slot(insn); } +NOKPROBE_SYMBOL(insn_has_delayslot); /* * insn_has_ll_or_sc function checks whether instruction is ll or sc * one; putting breakpoint on top of atomic ll/sc pair is bad idea; * so we need to prevent it and refuse kprobes insertion for such * instructions; cannot do much about breakpoint in the middle of - * ll/sc pair; it is upto user to avoid those places + * ll/sc pair; it is up to user to avoid those places */ -static int __kprobes insn_has_ll_or_sc(union mips_instruction insn) +static int insn_has_ll_or_sc(union mips_instruction insn) { int ret = 0; @@ -137,8 +73,9 @@ static int __kprobes insn_has_ll_or_sc(union mips_instruction insn) } return ret; } +NOKPROBE_SYMBOL(insn_has_ll_or_sc); -int __kprobes arch_prepare_kprobe(struct kprobe *p) +int arch_prepare_kprobe(struct kprobe *p) { union mips_instruction insn; union mips_instruction prev_insn; @@ -147,20 +84,25 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) insn = p->addr[0]; if (insn_has_ll_or_sc(insn)) { - pr_notice("Kprobes for ll and sc instructions are not" - "supported\n"); + pr_notice("Kprobes for ll and sc instructions are not supported\n"); ret = -EINVAL; goto out; } - if ((probe_kernel_read(&prev_insn, p->addr - 1, - sizeof(mips_instruction)) == 0) && - insn_has_delayslot(prev_insn)) { + if (copy_from_kernel_nofault(&prev_insn, p->addr - 1, + sizeof(mips_instruction)) == 0 && + insn_has_delayslot(prev_insn)) { pr_notice("Kprobes for branch delayslot are not supported\n"); ret = -EINVAL; goto out; } + if (__insn_is_compact_branch(insn)) { + pr_notice("Kprobes for compact branches are not supported\n"); + ret = -EINVAL; + goto out; + } + /* insn: must be on special executable page on mips. */ p->ainsn.insn = get_insn_slot(); if (!p->ainsn.insn) { @@ -192,26 +134,30 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) out: return ret; } +NOKPROBE_SYMBOL(arch_prepare_kprobe); -void __kprobes arch_arm_kprobe(struct kprobe *p) +void arch_arm_kprobe(struct kprobe *p) { *p->addr = breakpoint_insn; flush_insn_slot(p); } +NOKPROBE_SYMBOL(arch_arm_kprobe); -void __kprobes arch_disarm_kprobe(struct kprobe *p) +void arch_disarm_kprobe(struct kprobe *p) { *p->addr = p->opcode; flush_insn_slot(p); } +NOKPROBE_SYMBOL(arch_disarm_kprobe); -void __kprobes arch_remove_kprobe(struct kprobe *p) +void arch_remove_kprobe(struct kprobe *p) { if (p->ainsn.insn) { free_insn_slot(p->ainsn.insn, 0); p->ainsn.insn = NULL; } } +NOKPROBE_SYMBOL(arch_remove_kprobe); static void save_previous_kprobe(struct kprobe_ctlblk *kcb) { @@ -224,7 +170,7 @@ static void save_previous_kprobe(struct kprobe_ctlblk *kcb) static void restore_previous_kprobe(struct kprobe_ctlblk *kcb) { - __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp; + __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status; kcb->kprobe_old_SR = kcb->prev_kprobe.old_SR; kcb->kprobe_saved_SR = kcb->prev_kprobe.saved_SR; @@ -234,7 +180,7 @@ static void restore_previous_kprobe(struct kprobe_ctlblk *kcb) static void set_current_kprobe(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb) { - __get_cpu_var(current_kprobe) = p; + __this_cpu_write(current_kprobe, p); kcb->kprobe_saved_SR = kcb->kprobe_old_SR = (regs->cp0_status & ST0_IE); kcb->kprobe_saved_epc = regs->cp0_epc; } @@ -280,8 +226,8 @@ static int evaluate_branch_instruction(struct kprobe *p, struct pt_regs *regs, return 0; unaligned: - pr_notice("%s: unaligned epc - sending SIGBUS.\n", current->comm); - force_sig(SIGBUS, current); + pr_notice("Failed to emulate branch instruction because of unaligned epc - sending SIGBUS to %s.\n", current->comm); + force_sig(SIGBUS); return -EFAULT; } @@ -299,10 +245,8 @@ static void prepare_singlestep(struct kprobe *p, struct pt_regs *regs, regs->cp0_epc = (unsigned long)p->addr; else if (insn_has_delayslot(p->opcode)) { ret = evaluate_branch_instruction(p, regs, kcb); - if (ret < 0) { - pr_notice("Kprobes: Error in evaluating branch\n"); + if (ret < 0) return; - } } regs->cp0_epc = (unsigned long)&p->ainsn.insn[0]; } @@ -319,7 +263,7 @@ static void prepare_singlestep(struct kprobe *p, struct pt_regs *regs, * breakpoint trap. In case of branch instructions, the target * epc to be restored. */ -static void __kprobes resume_execution(struct kprobe *p, +static void resume_execution(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb) { @@ -330,8 +274,9 @@ static void __kprobes resume_execution(struct kprobe *p, regs->cp0_epc = orig_epc + 4; } } +NOKPROBE_SYMBOL(resume_execution); -static int __kprobes kprobe_handler(struct pt_regs *regs) +static int kprobe_handler(struct pt_regs *regs) { struct kprobe *p; int ret = 0; @@ -375,19 +320,13 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) preempt_enable_no_resched(); } return 1; - } else { - if (addr->word != breakpoint_insn.word) { - /* - * The breakpoint instruction was removed by - * another cpu right after we hit, no further - * handling of this interrupt is appropriate - */ - ret = 1; - goto no_kprobe; - } - p = __get_cpu_var(current_kprobe); - if (p->break_handler && p->break_handler(p, regs)) - goto ss_probe; + } else if (addr->word != breakpoint_insn.word) { + /* + * The breakpoint instruction was removed by + * another cpu right after we hit, no further + * handling of this interrupt is appropriate + */ + ret = 1; } goto no_kprobe; } @@ -413,10 +352,11 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) if (p->pre_handler && p->pre_handler(p, regs)) { /* handler has already set things up, so skip ss setup */ + reset_current_kprobe(); + preempt_enable_no_resched(); return 1; } -ss_probe: prepare_singlestep(p, regs, kcb); if (kcb->flags & SKIP_DELAYSLOT) { kcb->kprobe_status = KPROBE_HIT_SSDONE; @@ -434,6 +374,7 @@ no_kprobe: return ret; } +NOKPROBE_SYMBOL(kprobe_handler); static inline int post_kprobe_handler(struct pt_regs *regs) { @@ -464,14 +405,11 @@ out: return 1; } -static inline int kprobe_fault_handler(struct pt_regs *regs, int trapnr) +int kprobe_fault_handler(struct pt_regs *regs, int trapnr) { struct kprobe *cur = kprobe_running(); struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) - return 1; - if (kcb->kprobe_status & KPROBE_HIT_SS) { resume_execution(cur, regs, kcb); regs->cp0_status |= kcb->kprobe_old_SR; @@ -485,7 +423,7 @@ static inline int kprobe_fault_handler(struct pt_regs *regs, int trapnr) /* * Wrapper routine for handling exceptions. */ -int __kprobes kprobe_exceptions_notify(struct notifier_block *self, +int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *data) { @@ -516,51 +454,7 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, } return ret; } - -int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct jprobe *jp = container_of(p, struct jprobe, kp); - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - kcb->jprobe_saved_regs = *regs; - kcb->jprobe_saved_sp = regs->regs[29]; - - memcpy(kcb->jprobes_stack, (void *)kcb->jprobe_saved_sp, - MIN_JPROBES_STACK_SIZE(kcb->jprobe_saved_sp)); - - regs->cp0_epc = (unsigned long)(jp->entry); - - return 1; -} - -/* Defined in the inline asm below. */ -void jprobe_return_end(void); - -void __kprobes jprobe_return(void) -{ - /* Assembler quirk necessitates this '0,code' business. */ - asm volatile( - "break 0,%0\n\t" - ".globl jprobe_return_end\n" - "jprobe_return_end:\n" - : : "n" (BRK_KPROBE_BP) : "memory"); -} - -int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) -{ - struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); - - if (regs->cp0_epc >= (unsigned long)jprobe_return && - regs->cp0_epc <= (unsigned long)jprobe_return_end) { - *regs = kcb->jprobe_saved_regs; - memcpy((void *)kcb->jprobe_saved_sp, kcb->jprobes_stack, - MIN_JPROBES_STACK_SIZE(kcb->jprobe_saved_sp)); - preempt_enable_no_resched(); - - return 1; - } - return 0; -} +NOKPROBE_SYMBOL(kprobe_exceptions_notify); /* * Function return probe trampoline: @@ -575,83 +469,33 @@ static void __used kretprobe_trampoline_holder(void) /* Keep the assembler from reordering and placing JR here. */ ".set noreorder\n\t" "nop\n\t" - ".global kretprobe_trampoline\n" - "kretprobe_trampoline:\n\t" + ".global __kretprobe_trampoline\n" + "__kretprobe_trampoline:\n\t" "nop\n\t" ".set pop" : : : "memory"); } -void kretprobe_trampoline(void); +void __kretprobe_trampoline(void); -void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, +void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) { ri->ret_addr = (kprobe_opcode_t *) regs->regs[31]; + ri->fp = NULL; /* Replace the return addr with trampoline addr */ - regs->regs[31] = (unsigned long)kretprobe_trampoline; + regs->regs[31] = (unsigned long)__kretprobe_trampoline; } +NOKPROBE_SYMBOL(arch_prepare_kretprobe); /* * Called when the probe at kretprobe trampoline is hit */ -static int __kprobes trampoline_probe_handler(struct kprobe *p, +static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) { - struct kretprobe_instance *ri = NULL; - struct hlist_head *head, empty_rp; - struct hlist_node *tmp; - unsigned long flags, orig_ret_address = 0; - unsigned long trampoline_address = (unsigned long)kretprobe_trampoline; - - INIT_HLIST_HEAD(&empty_rp); - kretprobe_hash_lock(current, &head, &flags); - - /* - * It is possible to have multiple instances associated with a given - * task either because an multiple functions in the call path - * have a return probe installed on them, and/or more than one return - * return probe was registered for a target function. - * - * We can handle this because: - * - instances are always inserted at the head of the list - * - when multiple return probes are registered for the same - * function, the first instance's ret_addr will point to the - * real return address, and all the rest will point to - * kretprobe_trampoline - */ - hlist_for_each_entry_safe(ri, tmp, head, hlist) { - if (ri->task != current) - /* another task is sharing our hash bucket */ - continue; - - if (ri->rp && ri->rp->handler) - ri->rp->handler(ri, regs); - - orig_ret_address = (unsigned long)ri->ret_addr; - recycle_rp_inst(ri, &empty_rp); - - if (orig_ret_address != trampoline_address) - /* - * This is the real return address. Any other - * instances associated with this task are for - * other calls deeper on the call stack - */ - break; - } - - kretprobe_assert(ri, orig_ret_address, trampoline_address); - instruction_pointer(regs) = orig_ret_address; - - reset_current_kprobe(); - kretprobe_hash_unlock(current, &flags); - preempt_enable_no_resched(); - - hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { - hlist_del(&ri->hlist); - kfree(ri); - } + instruction_pointer(regs) = __kretprobe_trampoline_handler(regs, NULL); /* * By returning a non-zero value, we are telling * kprobe_handler() that we don't want the post_handler @@ -659,17 +503,19 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, */ return 1; } +NOKPROBE_SYMBOL(trampoline_probe_handler); -int __kprobes arch_trampoline_kprobe(struct kprobe *p) +int arch_trampoline_kprobe(struct kprobe *p) { - if (p->addr == (kprobe_opcode_t *)kretprobe_trampoline) + if (p->addr == (kprobe_opcode_t *)__kretprobe_trampoline) return 1; return 0; } +NOKPROBE_SYMBOL(arch_trampoline_kprobe); static struct kprobe trampoline_p = { - .addr = (kprobe_opcode_t *)kretprobe_trampoline, + .addr = (kprobe_opcode_t *)__kretprobe_trampoline, .pre_handler = trampoline_probe_handler }; diff --git a/arch/mips/kernel/linux32.c b/arch/mips/kernel/linux32.c index 0b29646bcee7..a0c0a7a654e9 100644 --- a/arch/mips/kernel/linux32.c +++ b/arch/mips/kernel/linux32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Conversion between 32-bit and 64-bit native system calls. * @@ -26,7 +27,6 @@ #include <linux/utsname.h> #include <linux/personality.h> #include <linux/dnotify.h> -#include <linux/module.h> #include <linux/binfmts.h> #include <linux/security.h> #include <linux/compat.h> @@ -39,20 +39,10 @@ #include <asm/compat-signal.h> #include <asm/sim.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/mmu_context.h> #include <asm/mman.h> - -/* Use this to get at 32-bit user passed pointers. */ -/* A() macro should be used for places where you e.g. - have some internal variable u32 and just want to get - rid of a compiler warning. AA() has to be used in - places where you want to convert a function argument - to 32bit pointer or when you e.g. access pt_regs - structure and want to consider 32bit registers only. - */ -#define A(__x) ((unsigned long)(__x)) -#define AA(__x) ((unsigned long)((int)__x)) +#include <asm/syscalls.h> #ifdef __MIPSEB__ #define merge_64(r1, r2) ((((r1) & 0xffffffffUL) << 32) + ((r2) & 0xffffffffUL)) @@ -61,39 +51,16 @@ #define merge_64(r1, r2) ((((r2) & 0xffffffffUL) << 32) + ((r1) & 0xffffffffUL)) #endif -SYSCALL_DEFINE6(32_mmap2, unsigned long, addr, unsigned long, len, - unsigned long, prot, unsigned long, flags, unsigned long, fd, - unsigned long, pgoff) -{ - unsigned long error; - - error = -EINVAL; - if (pgoff & (~PAGE_MASK >> 12)) - goto out; - error = sys_mmap_pgoff(addr, len, prot, flags, fd, - pgoff >> (PAGE_SHIFT-12)); -out: - return error; -} - -#define RLIM_INFINITY32 0x7fffffff -#define RESOURCE32(x) ((x > RLIM_INFINITY32) ? RLIM_INFINITY32 : x) - -struct rlimit32 { - int rlim_cur; - int rlim_max; -}; - SYSCALL_DEFINE4(32_truncate64, const char __user *, path, unsigned long, __dummy, unsigned long, a2, unsigned long, a3) { - return sys_truncate(path, merge_64(a2, a3)); + return ksys_truncate(path, merge_64(a2, a3)); } SYSCALL_DEFINE4(32_ftruncate64, unsigned long, fd, unsigned long, __dummy, unsigned long, a2, unsigned long, a3) { - return sys_ftruncate(fd, merge_64(a2, a3)); + return ksys_ftruncate(fd, merge_64(a2, a3)); } SYSCALL_DEFINE5(32_llseek, unsigned int, fd, unsigned int, offset_high, @@ -110,13 +77,13 @@ SYSCALL_DEFINE5(32_llseek, unsigned int, fd, unsigned int, offset_high, SYSCALL_DEFINE6(32_pread, unsigned long, fd, char __user *, buf, size_t, count, unsigned long, unused, unsigned long, a4, unsigned long, a5) { - return sys_pread64(fd, buf, count, merge_64(a4, a5)); + return ksys_pread64(fd, buf, count, merge_64(a4, a5)); } SYSCALL_DEFINE6(32_pwrite, unsigned int, fd, const char __user *, buf, size_t, count, u32, unused, u64, a4, u64, a5) { - return sys_pwrite64(fd, buf, count, merge_64(a4, a5)); + return ksys_pwrite64(fd, buf, count, merge_64(a4, a5)); } SYSCALL_DEFINE1(32_personality, unsigned long, personality) @@ -136,7 +103,7 @@ SYSCALL_DEFINE1(32_personality, unsigned long, personality) asmlinkage ssize_t sys32_readahead(int fd, u32 pad0, u64 a2, u64 a3, size_t count) { - return sys_readahead(fd, merge_64(a2, a3), count); + return ksys_readahead(fd, merge_64(a2, a3), count); } asmlinkage long sys32_sync_file_range(int fd, int __pad, @@ -144,7 +111,7 @@ asmlinkage long sys32_sync_file_range(int fd, int __pad, unsigned long a4, unsigned long a5, int flags) { - return sys_sync_file_range(fd, + return ksys_sync_file_range(fd, merge_64(a2, a3), merge_64(a4, a5), flags); } @@ -154,7 +121,7 @@ asmlinkage long sys32_fadvise64_64(int fd, int __pad, unsigned long a4, unsigned long a5, int flags) { - return sys_fadvise64_64(fd, + return ksys_fadvise64_64(fd, merge_64(a2, a3), merge_64(a4, a5), flags); } @@ -162,6 +129,6 @@ asmlinkage long sys32_fadvise64_64(int fd, int __pad, asmlinkage long sys32_fallocate(int fd, int mode, unsigned offset_a2, unsigned offset_a3, unsigned len_a4, unsigned len_a5) { - return sys_fallocate(fd, mode, merge_64(offset_a2, offset_a3), - merge_64(len_a4, len_a5)); + return ksys_fallocate(fd, mode, merge_64(offset_a2, offset_a3), + merge_64(len_a4, len_a5)); } diff --git a/arch/mips/kernel/machine_kexec.c b/arch/mips/kernel/machine_kexec.c index 992e18474da5..4e3579bbd620 100644 --- a/arch/mips/kernel/machine_kexec.c +++ b/arch/mips/kernel/machine_kexec.c @@ -1,14 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * machine_kexec.c for kexec * Created by <nschichan@corp.free.fr> on Thu Oct 12 15:15:06 2006 - * - * This source code is licensed under the GNU General Public License, - * Version 2. See the file COPYING for more details. */ #include <linux/compiler.h> #include <linux/kexec.h> #include <linux/mm.h> #include <linux/delay.h> +#include <linux/libfdt.h> +#include <linux/reboot.h> #include <asm/cacheflush.h> #include <asm/page.h> @@ -19,19 +19,91 @@ extern const size_t relocate_new_kernel_size; extern unsigned long kexec_start_address; extern unsigned long kexec_indirection_page; -int (*_machine_kexec_prepare)(struct kimage *) = NULL; -void (*_machine_kexec_shutdown)(void) = NULL; -void (*_machine_crash_shutdown)(struct pt_regs *regs) = NULL; +static unsigned long reboot_code_buffer; + #ifdef CONFIG_SMP -void (*relocated_kexec_smp_wait) (void *); +static void (*relocated_kexec_smp_wait)(void *); + atomic_t kexec_ready_to_reboot = ATOMIC_INIT(0); +void (*_crash_smp_send_stop)(void) = NULL; #endif +void (*_machine_kexec_shutdown)(void) = NULL; +void (*_machine_crash_shutdown)(struct pt_regs *regs) = NULL; + +static void kexec_image_info(const struct kimage *kimage) +{ + unsigned long i; + + pr_debug("kexec kimage info:\n"); + pr_debug(" type: %d\n", kimage->type); + pr_debug(" start: %lx\n", kimage->start); + pr_debug(" head: %lx\n", kimage->head); + pr_debug(" nr_segments: %lu\n", kimage->nr_segments); + + for (i = 0; i < kimage->nr_segments; i++) { + pr_debug(" segment[%lu]: %016lx - %016lx, 0x%lx bytes, %lu pages\n", + i, + kimage->segment[i].mem, + kimage->segment[i].mem + kimage->segment[i].memsz, + (unsigned long)kimage->segment[i].memsz, + (unsigned long)kimage->segment[i].memsz / PAGE_SIZE); + } +} + +#ifdef CONFIG_UHI_BOOT + +static int uhi_machine_kexec_prepare(struct kimage *kimage) +{ + int i; + + /* + * In case DTB file is not passed to the new kernel, a flat device + * tree will be created by kexec tool. It holds modified command + * line for the new kernel. + */ + for (i = 0; i < kimage->nr_segments; i++) { + struct fdt_header fdt; + + if (kimage->segment[i].memsz <= sizeof(fdt)) + continue; + + if (copy_from_user(&fdt, kimage->segment[i].buf, sizeof(fdt))) + continue; + + if (fdt_check_header(&fdt)) + continue; + + kexec_args[0] = -2; + kexec_args[1] = (unsigned long) + phys_to_virt((unsigned long)kimage->segment[i].mem); + break; + } + + return 0; +} + +int (*_machine_kexec_prepare)(struct kimage *) = uhi_machine_kexec_prepare; + +#else + +int (*_machine_kexec_prepare)(struct kimage *) = NULL; + +#endif /* CONFIG_UHI_BOOT */ + int machine_kexec_prepare(struct kimage *kimage) { +#ifdef CONFIG_SMP + if (!kexec_nonboot_cpu_func()) + return -EINVAL; +#endif + + kexec_image_info(kimage); + if (_machine_kexec_prepare) return _machine_kexec_prepare(kimage); + return 0; } @@ -40,11 +112,41 @@ machine_kexec_cleanup(struct kimage *kimage) { } +#ifdef CONFIG_SMP +static void kexec_shutdown_secondary(void *param) +{ + int cpu = smp_processor_id(); + + if (!cpu_online(cpu)) + return; + + /* We won't be sent IPIs any more. */ + set_cpu_online(cpu, false); + + local_irq_disable(); + while (!atomic_read(&kexec_ready_to_reboot)) + cpu_relax(); + + kexec_reboot(); + + /* NOTREACHED */ +} +#endif + void machine_shutdown(void) { if (_machine_kexec_shutdown) _machine_kexec_shutdown(); + +#ifdef CONFIG_SMP + smp_call_function(kexec_shutdown_secondary, NULL, 0); + + while (num_online_cpus() > 1) { + cpu_relax(); + mdelay(1); + } +#endif } void @@ -56,12 +158,57 @@ machine_crash_shutdown(struct pt_regs *regs) default_machine_crash_shutdown(regs); } -typedef void (*noretfun_t)(void) __noreturn; +#ifdef CONFIG_SMP +void kexec_nonboot_cpu_jump(void) +{ + local_flush_icache_range((unsigned long)relocated_kexec_smp_wait, + reboot_code_buffer + relocate_new_kernel_size); + + relocated_kexec_smp_wait(NULL); +} +#endif + +void kexec_reboot(void) +{ + void (*do_kexec)(void) __noreturn; + + /* + * We know we were online, and there will be no incoming IPIs at + * this point. Mark online again before rebooting so that the crash + * analysis tool will see us correctly. + */ + set_cpu_online(smp_processor_id(), true); + + /* Ensure remote CPUs observe that we're online before rebooting. */ + smp_mb__after_atomic(); + +#ifdef CONFIG_SMP + if (smp_processor_id() > 0) { + /* + * Instead of cpu_relax() or wait, this is needed for kexec + * smp reboot. Kdump usually doesn't require an smp new + * kernel, but kexec may do. + */ + kexec_nonboot_cpu(); + + /* NOTREACHED */ + } +#endif + + /* + * Make sure we get correct instructions written by the + * machine_kexec() CPU. + */ + local_flush_icache_range(reboot_code_buffer, + reboot_code_buffer + relocate_new_kernel_size); + + do_kexec = (void *)reboot_code_buffer; + do_kexec(); +} void machine_kexec(struct kimage *image) { - unsigned long reboot_code_buffer; unsigned long entry; unsigned long *ptr; @@ -71,8 +218,12 @@ machine_kexec(struct kimage *image) kexec_start_address = (unsigned long) phys_to_virt(image->start); - kexec_indirection_page = - (unsigned long) phys_to_virt(image->head & PAGE_MASK); + if (image->type == KEXEC_TYPE_DEFAULT) { + kexec_indirection_page = + (unsigned long) phys_to_virt(image->head & PAGE_MASK); + } else { + kexec_indirection_page = (unsigned long)&image->head; + } memcpy((void*)reboot_code_buffer, relocate_new_kernel, relocate_new_kernel_size); @@ -91,6 +242,9 @@ machine_kexec(struct kimage *image) *ptr = (unsigned long) phys_to_virt(*ptr); } + /* Mark offline BEFORE disabling local irq. */ + set_cpu_online(smp_processor_id(), false); + /* * we do not want to be bothered. */ @@ -98,6 +252,7 @@ machine_kexec(struct kimage *image) printk("Will call new kernel at %08lx\n", image->start); printk("Bye ...\n"); + /* Make reboot code buffer available to the boot CPU. */ __flush_cache_all(); #ifdef CONFIG_SMP /* All secondary cpus now may jump to kexec_wait cycle */ @@ -106,5 +261,5 @@ machine_kexec(struct kimage *image) smp_wmb(); atomic_set(&kexec_ready_to_reboot, 1); #endif - ((noretfun_t) reboot_code_buffer)(); + kexec_reboot(); } diff --git a/arch/mips/kernel/mcount.S b/arch/mips/kernel/mcount.S index a03e93c4a946..fcec579f64e9 100644 --- a/arch/mips/kernel/mcount.S +++ b/arch/mips/kernel/mcount.S @@ -10,6 +10,7 @@ * Author: Wu Zhangjin <wuzhangjin@gmail.com> */ +#include <linux/export.h> #include <asm/regdef.h> #include <asm/stackframe.h> #include <asm/ftrace.h> @@ -66,6 +67,7 @@ NESTED(ftrace_caller, PT_SIZE, ra) .globl _mcount _mcount: +EXPORT_SYMBOL(_mcount) b ftrace_stub #ifdef CONFIG_32BIT addiu sp,sp,8 @@ -74,16 +76,25 @@ _mcount: #endif /* When tracing is activated, it calls ftrace_caller+8 (aka here) */ - lw t1, function_trace_stop - bnez t1, ftrace_stub - nop - MCOUNT_SAVE_REGS #ifdef KBUILD_MCOUNT_RA_ADDRESS PTR_S MCOUNT_RA_ADDRESS_REG, PT_R12(sp) #endif - move a0, ra /* arg1: self return address */ + PTR_SUBU a0, ra, 8 /* arg1: self address */ + PTR_LA t1, _stext + sltu t2, a0, t1 /* t2 = (a0 < _stext) */ + PTR_LA t1, _etext + sltu t3, t1, a0 /* t3 = (a0 > _etext) */ + or t1, t2, t3 + beqz t1, ftrace_call + nop +#if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT) + PTR_SUBU a0, a0, 16 /* arg1: adjust to module's recorded callsite */ +#else + PTR_SUBU a0, a0, 12 +#endif + .globl ftrace_call ftrace_call: nop /* a placeholder for the call to a real tracing function */ @@ -105,15 +116,23 @@ ftrace_stub: #else /* ! CONFIG_DYNAMIC_FTRACE */ NESTED(_mcount, PT_SIZE, ra) - lw t1, function_trace_stop - bnez t1, ftrace_stub - nop +EXPORT_SYMBOL(_mcount) PTR_LA t1, ftrace_stub PTR_L t2, ftrace_trace_function /* Prepare t2 for (1) */ - bne t1, t2, static_trace + beq t1, t2, fgraph_trace nop + MCOUNT_SAVE_REGS + + move a0, ra /* arg1: self return address */ + jalr t2 /* (1) call *ftrace_trace_function */ + move a1, AT /* arg2: parent's return address */ + + MCOUNT_RESTORE_REGS + +fgraph_trace: #ifdef CONFIG_FUNCTION_GRAPH_TRACER + PTR_LA t1, ftrace_stub PTR_L t3, ftrace_graph_return bne t1, t3, ftrace_graph_caller nop @@ -122,17 +141,11 @@ NESTED(_mcount, PT_SIZE, ra) bne t1, t3, ftrace_graph_caller nop #endif - b ftrace_stub - nop - -static_trace: - MCOUNT_SAVE_REGS - move a0, ra /* arg1: self return address */ - jalr t2 /* (1) call *ftrace_trace_function */ - move a1, AT /* arg2: parent's return address */ +#ifdef CONFIG_32BIT + addiu sp, sp, 8 +#endif - MCOUNT_RESTORE_REGS .globl ftrace_stub ftrace_stub: RETURN_BACK @@ -177,6 +190,11 @@ NESTED(ftrace_graph_caller, PT_SIZE, ra) jal prepare_ftrace_return nop MCOUNT_RESTORE_REGS +#ifndef CONFIG_DYNAMIC_FTRACE +#ifdef CONFIG_32BIT + addiu sp, sp, 8 +#endif +#endif RETURN_BACK END(ftrace_graph_caller) diff --git a/arch/mips/kernel/mips-cm.c b/arch/mips/kernel/mips-cm.c new file mode 100644 index 000000000000..7c9c5dc38823 --- /dev/null +++ b/arch/mips/kernel/mips-cm.c @@ -0,0 +1,557 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2013 Imagination Technologies + * Author: Paul Burton <paul.burton@mips.com> + */ + +#include <linux/errno.h> +#include <linux/of.h> +#include <linux/percpu.h> +#include <linux/spinlock.h> + +#include <asm/mips-cps.h> +#include <asm/smp-cps.h> +#include <asm/mipsregs.h> + +void __iomem *mips_gcr_base; +void __iomem *mips_cm_l2sync_base; +int mips_cm_is64; +bool mips_cm_is_l2_hci_broken; + +static char *cm2_tr[8] = { + "mem", "gcr", "gic", "mmio", + "0x04", "cpc", "0x06", "0x07" +}; + +/* CM3 Tag ECC transaction type */ +static char *cm3_tr[16] = { + [0x0] = "ReqNoData", + [0x1] = "0x1", + [0x2] = "ReqWData", + [0x3] = "0x3", + [0x4] = "IReqNoResp", + [0x5] = "IReqWResp", + [0x6] = "IReqNoRespDat", + [0x7] = "IReqWRespDat", + [0x8] = "RespNoData", + [0x9] = "RespDataFol", + [0xa] = "RespWData", + [0xb] = "RespDataOnly", + [0xc] = "IRespNoData", + [0xd] = "IRespDataFol", + [0xe] = "IRespWData", + [0xf] = "IRespDataOnly" +}; + +static char *cm2_cmd[32] = { + [0x00] = "0x00", + [0x01] = "Legacy Write", + [0x02] = "Legacy Read", + [0x03] = "0x03", + [0x04] = "0x04", + [0x05] = "0x05", + [0x06] = "0x06", + [0x07] = "0x07", + [0x08] = "Coherent Read Own", + [0x09] = "Coherent Read Share", + [0x0a] = "Coherent Read Discard", + [0x0b] = "Coherent Ready Share Always", + [0x0c] = "Coherent Upgrade", + [0x0d] = "Coherent Writeback", + [0x0e] = "0x0e", + [0x0f] = "0x0f", + [0x10] = "Coherent Copyback", + [0x11] = "Coherent Copyback Invalidate", + [0x12] = "Coherent Invalidate", + [0x13] = "Coherent Write Invalidate", + [0x14] = "Coherent Completion Sync", + [0x15] = "0x15", + [0x16] = "0x16", + [0x17] = "0x17", + [0x18] = "0x18", + [0x19] = "0x19", + [0x1a] = "0x1a", + [0x1b] = "0x1b", + [0x1c] = "0x1c", + [0x1d] = "0x1d", + [0x1e] = "0x1e", + [0x1f] = "0x1f" +}; + +/* CM3 Tag ECC command type */ +static char *cm3_cmd[16] = { + [0x0] = "Legacy Read", + [0x1] = "Legacy Write", + [0x2] = "Coherent Read Own", + [0x3] = "Coherent Read Share", + [0x4] = "Coherent Read Discard", + [0x5] = "Coherent Evicted", + [0x6] = "Coherent Upgrade", + [0x7] = "Coherent Upgrade for Store Conditional", + [0x8] = "Coherent Writeback", + [0x9] = "Coherent Write Invalidate", + [0xa] = "0xa", + [0xb] = "0xb", + [0xc] = "0xc", + [0xd] = "0xd", + [0xe] = "0xe", + [0xf] = "0xf" +}; + +/* CM3 Tag ECC command group */ +static char *cm3_cmd_group[8] = { + [0x0] = "Normal", + [0x1] = "Registers", + [0x2] = "TLB", + [0x3] = "0x3", + [0x4] = "L1I", + [0x5] = "L1D", + [0x6] = "L3", + [0x7] = "L2" +}; + +static char *cm2_core[8] = { + "Invalid/OK", "Invalid/Data", + "Shared/OK", "Shared/Data", + "Modified/OK", "Modified/Data", + "Exclusive/OK", "Exclusive/Data" +}; + +static char *cm2_l2_type[4] = { + [0x0] = "None", + [0x1] = "Tag RAM single/double ECC error", + [0x2] = "Data RAM single/double ECC error", + [0x3] = "WS RAM uncorrectable dirty parity" +}; + +static char *cm2_l2_instr[32] = { + [0x00] = "L2_NOP", + [0x01] = "L2_ERR_CORR", + [0x02] = "L2_TAG_INV", + [0x03] = "L2_WS_CLEAN", + [0x04] = "L2_RD_MDYFY_WR", + [0x05] = "L2_WS_MRU", + [0x06] = "L2_EVICT_LN2", + [0x07] = "0x07", + [0x08] = "L2_EVICT", + [0x09] = "L2_REFL", + [0x0a] = "L2_RD", + [0x0b] = "L2_WR", + [0x0c] = "L2_EVICT_MRU", + [0x0d] = "L2_SYNC", + [0x0e] = "L2_REFL_ERR", + [0x0f] = "0x0f", + [0x10] = "L2_INDX_WB_INV", + [0x11] = "L2_INDX_LD_TAG", + [0x12] = "L2_INDX_ST_TAG", + [0x13] = "L2_INDX_ST_DATA", + [0x14] = "L2_INDX_ST_ECC", + [0x15] = "0x15", + [0x16] = "0x16", + [0x17] = "0x17", + [0x18] = "L2_FTCH_AND_LCK", + [0x19] = "L2_HIT_INV", + [0x1a] = "L2_HIT_WB_INV", + [0x1b] = "L2_HIT_WB", + [0x1c] = "0x1c", + [0x1d] = "0x1d", + [0x1e] = "0x1e", + [0x1f] = "0x1f" +}; + +static char *cm2_causes[32] = { + "None", "GC_WR_ERR", "GC_RD_ERR", "COH_WR_ERR", + "COH_RD_ERR", "MMIO_WR_ERR", "MMIO_RD_ERR", "0x07", + "0x08", "0x09", "0x0a", "0x0b", + "0x0c", "0x0d", "0x0e", "0x0f", + "0x10", "INTVN_WR_ERR", "INTVN_RD_ERR", "0x13", + "0x14", "0x15", "0x16", "0x17", + "L2_RD_UNCORR", "L2_WR_UNCORR", "L2_CORR", "0x1b", + "0x1c", "0x1d", "0x1e", "0x1f" +}; + +static char *cm3_causes[32] = { + "0x0", "MP_CORRECTABLE_ECC_ERR", "MP_REQUEST_DECODE_ERR", + "MP_UNCORRECTABLE_ECC_ERR", "MP_PARITY_ERR", "MP_COHERENCE_ERR", + "CMBIU_REQUEST_DECODE_ERR", "CMBIU_PARITY_ERR", "CMBIU_AXI_RESP_ERR", + "0x9", "RBI_BUS_ERR", "0xb", "0xc", "0xd", "0xe", "0xf", "0x10", + "0x11", "0x12", "0x13", "0x14", "0x15", "0x16", "0x17", "0x18", + "0x19", "0x1a", "0x1b", "0x1c", "0x1d", "0x1e", "0x1f" +}; + +static DEFINE_PER_CPU_ALIGNED(spinlock_t, cm_core_lock); +static DEFINE_PER_CPU_ALIGNED(unsigned long, cm_core_lock_flags); + +phys_addr_t __weak mips_cm_phys_base(void) +{ + unsigned long cmgcr; + + /* Check the CMGCRBase register is implemented */ + if (!(read_c0_config() & MIPS_CONF_M)) + return 0; + + if (!(read_c0_config2() & MIPS_CONF_M)) + return 0; + + if (!(read_c0_config3() & MIPS_CONF3_CMGCR)) + return 0; + + /* Read the address from CMGCRBase */ + cmgcr = read_c0_cmgcrbase(); + return (cmgcr & MIPS_CMGCRF_BASE) << (36 - 32); +} + +phys_addr_t __weak mips_cm_l2sync_phys_base(void) +{ + u32 base_reg; + + /* + * If the L2-only sync region is already enabled then leave it at it's + * current location. + */ + base_reg = read_gcr_l2_only_sync_base(); + if (base_reg & CM_GCR_L2_ONLY_SYNC_BASE_SYNCEN) + return base_reg & CM_GCR_L2_ONLY_SYNC_BASE_SYNCBASE; + + /* Default to following the CM */ + return mips_cm_phys_base() + MIPS_CM_GCR_SIZE; +} + +static void mips_cm_probe_l2sync(void) +{ + unsigned major_rev; + phys_addr_t addr; + + /* L2-only sync was introduced with CM major revision 6 */ + major_rev = FIELD_GET(CM_GCR_REV_MAJOR, read_gcr_rev()); + if (major_rev < 6) + return; + + /* Find a location for the L2 sync region */ + addr = mips_cm_l2sync_phys_base(); + BUG_ON((addr & CM_GCR_L2_ONLY_SYNC_BASE_SYNCBASE) != addr); + if (!addr) + return; + + /* Set the region base address & enable it */ + write_gcr_l2_only_sync_base(addr | CM_GCR_L2_ONLY_SYNC_BASE_SYNCEN); + + /* Map the region */ + mips_cm_l2sync_base = ioremap(addr, MIPS_CM_L2SYNC_SIZE); +} + +void mips_cm_update_property(void) +{ + struct device_node *cm_node; + + cm_node = of_find_compatible_node(of_root, NULL, "mobileye,eyeq6-cm"); + if (!cm_node) + return; + pr_info("HCI (Hardware Cache Init for the L2 cache) in GCR_L2_RAM_CONFIG from the CM3 is broken"); + mips_cm_is_l2_hci_broken = true; + + /* Disable MMID only if it was configured */ + if (cpu_has_mmid) + cpu_disable_mmid(); + + of_node_put(cm_node); +} + +int mips_cm_probe(void) +{ + phys_addr_t addr; + u32 base_reg; + unsigned cpu; + + /* + * No need to probe again if we have already been + * here before. + */ + if (mips_gcr_base) + return 0; + + addr = mips_cm_phys_base(); + BUG_ON((addr & CM_GCR_BASE_GCRBASE) != addr); + if (!addr) + return -ENODEV; + + mips_gcr_base = ioremap(addr, MIPS_CM_GCR_SIZE); + if (!mips_gcr_base) + return -ENXIO; + + /* sanity check that we're looking at a CM */ + base_reg = read_gcr_base(); + if ((base_reg & CM_GCR_BASE_GCRBASE) != addr) { + pr_err("GCRs appear to have been moved (expected them at 0x%08lx)!\n", + (unsigned long)addr); + iounmap(mips_gcr_base); + mips_gcr_base = NULL; + return -ENODEV; + } + + /* set default target to memory */ + change_gcr_base(CM_GCR_BASE_CMDEFTGT, CM_GCR_BASE_CMDEFTGT_MEM); + + /* disable CM regions */ + write_gcr_reg0_base(CM_GCR_REGn_BASE_BASEADDR); + write_gcr_reg0_mask(CM_GCR_REGn_MASK_ADDRMASK); + write_gcr_reg1_base(CM_GCR_REGn_BASE_BASEADDR); + write_gcr_reg1_mask(CM_GCR_REGn_MASK_ADDRMASK); + write_gcr_reg2_base(CM_GCR_REGn_BASE_BASEADDR); + write_gcr_reg2_mask(CM_GCR_REGn_MASK_ADDRMASK); + write_gcr_reg3_base(CM_GCR_REGn_BASE_BASEADDR); + write_gcr_reg3_mask(CM_GCR_REGn_MASK_ADDRMASK); + + /* probe for an L2-only sync region */ + mips_cm_probe_l2sync(); + + /* determine register width for this CM */ + mips_cm_is64 = IS_ENABLED(CONFIG_64BIT) && (mips_cm_revision() >= CM_REV_CM3); + + for_each_possible_cpu(cpu) + spin_lock_init(&per_cpu(cm_core_lock, cpu)); + + return 0; +} + +void mips_cm_lock_other(unsigned int cluster, unsigned int core, + unsigned int vp, unsigned int block) +{ + unsigned int curr_core, cm_rev; + u32 val; + + cm_rev = mips_cm_revision(); + preempt_disable(); + + if (cm_rev >= CM_REV_CM3) { + val = FIELD_PREP(CM3_GCR_Cx_OTHER_CORE, core) | + FIELD_PREP(CM3_GCR_Cx_OTHER_VP, vp); + + if (cm_rev >= CM_REV_CM3_5) { + if (cluster != cpu_cluster(¤t_cpu_data)) + val |= CM_GCR_Cx_OTHER_CLUSTER_EN; + val |= CM_GCR_Cx_OTHER_GIC_EN; + val |= FIELD_PREP(CM_GCR_Cx_OTHER_CLUSTER, cluster); + val |= FIELD_PREP(CM_GCR_Cx_OTHER_BLOCK, block); + } else { + WARN_ON(cluster != 0); + WARN_ON(block != CM_GCR_Cx_OTHER_BLOCK_LOCAL); + } + + /* + * We need to disable interrupts in SMP systems in order to + * ensure that we don't interrupt the caller with code which + * may modify the redirect register. We do so here in a + * slightly obscure way by using a spin lock, since this has + * the neat property of also catching any nested uses of + * mips_cm_lock_other() leading to a deadlock or a nice warning + * with lockdep enabled. + */ + spin_lock_irqsave(this_cpu_ptr(&cm_core_lock), + *this_cpu_ptr(&cm_core_lock_flags)); + } else { + WARN_ON(cluster != 0); + WARN_ON(block != CM_GCR_Cx_OTHER_BLOCK_LOCAL); + + /* + * We only have a GCR_CL_OTHER per core in systems with + * CM 2.5 & older, so have to ensure other VP(E)s don't + * race with us. + */ + curr_core = cpu_core(¤t_cpu_data); + spin_lock_irqsave(&per_cpu(cm_core_lock, curr_core), + per_cpu(cm_core_lock_flags, curr_core)); + + val = FIELD_PREP(CM_GCR_Cx_OTHER_CORENUM, core); + } + + write_gcr_cl_other(val); + + /* + * Ensure the core-other region reflects the appropriate core & + * VP before any accesses to it occur. + */ + mb(); +} + +void mips_cm_unlock_other(void) +{ + unsigned int curr_core; + + if (mips_cm_revision() < CM_REV_CM3) { + curr_core = cpu_core(¤t_cpu_data); + spin_unlock_irqrestore(&per_cpu(cm_core_lock, curr_core), + per_cpu(cm_core_lock_flags, curr_core)); + } else { + spin_unlock_irqrestore(this_cpu_ptr(&cm_core_lock), + *this_cpu_ptr(&cm_core_lock_flags)); + } + + preempt_enable(); +} + +void mips_cm_error_report(void) +{ + u64 cm_error, cm_addr, cm_other; + unsigned long revision; + int ocause, cause; + char buf[256]; + + if (!mips_cm_present()) + return; + + revision = mips_cm_revision(); + cm_error = read_gcr_error_cause(); + cm_addr = read_gcr_error_addr(); + cm_other = read_gcr_error_mult(); + + if (revision < CM_REV_CM3) { /* CM2 */ + cause = FIELD_GET(CM_GCR_ERROR_CAUSE_ERRTYPE, cm_error); + ocause = FIELD_GET(CM_GCR_ERROR_MULT_ERR2ND, cm_other); + + if (!cause) + return; + + if (cause < 16) { + unsigned long cca_bits = (cm_error >> 15) & 7; + unsigned long tr_bits = (cm_error >> 12) & 7; + unsigned long cmd_bits = (cm_error >> 7) & 0x1f; + unsigned long stag_bits = (cm_error >> 3) & 15; + unsigned long sport_bits = (cm_error >> 0) & 7; + + snprintf(buf, sizeof(buf), + "CCA=%lu TR=%s MCmd=%s STag=%lu " + "SPort=%lu\n", cca_bits, cm2_tr[tr_bits], + cm2_cmd[cmd_bits], stag_bits, sport_bits); + } else if (cause < 24) { + /* glob state & sresp together */ + unsigned long c3_bits = (cm_error >> 18) & 7; + unsigned long c2_bits = (cm_error >> 15) & 7; + unsigned long c1_bits = (cm_error >> 12) & 7; + unsigned long c0_bits = (cm_error >> 9) & 7; + unsigned long sc_bit = (cm_error >> 8) & 1; + unsigned long cmd_bits = (cm_error >> 3) & 0x1f; + unsigned long sport_bits = (cm_error >> 0) & 7; + + snprintf(buf, sizeof(buf), + "C3=%s C2=%s C1=%s C0=%s SC=%s " + "MCmd=%s SPort=%lu\n", + cm2_core[c3_bits], cm2_core[c2_bits], + cm2_core[c1_bits], cm2_core[c0_bits], + sc_bit ? "True" : "False", + cm2_cmd[cmd_bits], sport_bits); + } else { + unsigned long muc_bit = (cm_error >> 23) & 1; + unsigned long ins_bits = (cm_error >> 18) & 0x1f; + unsigned long arr_bits = (cm_error >> 16) & 3; + unsigned long dw_bits = (cm_error >> 12) & 15; + unsigned long way_bits = (cm_error >> 9) & 7; + unsigned long mway_bit = (cm_error >> 8) & 1; + unsigned long syn_bits = (cm_error >> 0) & 0xFF; + + snprintf(buf, sizeof(buf), + "Type=%s%s Instr=%s DW=%lu Way=%lu " + "MWay=%s Syndrome=0x%02lx", + muc_bit ? "Multi-UC " : "", + cm2_l2_type[arr_bits], + cm2_l2_instr[ins_bits], dw_bits, way_bits, + mway_bit ? "True" : "False", syn_bits); + } + pr_err("CM_ERROR=%08llx %s <%s>\n", cm_error, + cm2_causes[cause], buf); + pr_err("CM_ADDR =%08llx\n", cm_addr); + pr_err("CM_OTHER=%08llx %s\n", cm_other, cm2_causes[ocause]); + } else { /* CM3 */ + ulong core_id_bits, vp_id_bits, cmd_bits, cmd_group_bits; + ulong cm3_cca_bits, mcp_bits, cm3_tr_bits, sched_bit; + + cause = FIELD_GET(CM3_GCR_ERROR_CAUSE_ERRTYPE, cm_error); + ocause = FIELD_GET(CM_GCR_ERROR_MULT_ERR2ND, cm_other); + + if (!cause) + return; + + /* Used by cause == {1,2,3} */ + core_id_bits = (cm_error >> 22) & 0xf; + vp_id_bits = (cm_error >> 18) & 0xf; + cmd_bits = (cm_error >> 14) & 0xf; + cmd_group_bits = (cm_error >> 11) & 0xf; + cm3_cca_bits = (cm_error >> 8) & 7; + mcp_bits = (cm_error >> 5) & 0xf; + cm3_tr_bits = (cm_error >> 1) & 0xf; + sched_bit = cm_error & 0x1; + + if (cause == 1 || cause == 3) { /* Tag ECC */ + unsigned long tag_ecc = (cm_error >> 57) & 0x1; + unsigned long tag_way_bits = (cm_error >> 29) & 0xffff; + unsigned long dword_bits = (cm_error >> 49) & 0xff; + unsigned long data_way_bits = (cm_error >> 45) & 0xf; + unsigned long data_sets_bits = (cm_error >> 29) & 0xfff; + unsigned long bank_bit = (cm_error >> 28) & 0x1; + snprintf(buf, sizeof(buf), + "%s ECC Error: Way=%lu (DWORD=%lu, Sets=%lu)" + "Bank=%lu CoreID=%lu VPID=%lu Command=%s" + "Command Group=%s CCA=%lu MCP=%d" + "Transaction type=%s Scheduler=%lu\n", + tag_ecc ? "TAG" : "DATA", + tag_ecc ? (unsigned long)ffs(tag_way_bits) - 1 : + data_way_bits, bank_bit, dword_bits, + data_sets_bits, + core_id_bits, vp_id_bits, + cm3_cmd[cmd_bits], + cm3_cmd_group[cmd_group_bits], + cm3_cca_bits, 1 << mcp_bits, + cm3_tr[cm3_tr_bits], sched_bit); + } else if (cause == 2) { + unsigned long data_error_type = (cm_error >> 41) & 0xfff; + unsigned long data_decode_cmd = (cm_error >> 37) & 0xf; + unsigned long data_decode_group = (cm_error >> 34) & 0x7; + unsigned long data_decode_destination_id = (cm_error >> 28) & 0x3f; + + snprintf(buf, sizeof(buf), + "Decode Request Error: Type=%lu, Command=%lu" + "Command Group=%lu Destination ID=%lu" + "CoreID=%lu VPID=%lu Command=%s" + "Command Group=%s CCA=%lu MCP=%d" + "Transaction type=%s Scheduler=%lu\n", + data_error_type, data_decode_cmd, + data_decode_group, data_decode_destination_id, + core_id_bits, vp_id_bits, + cm3_cmd[cmd_bits], + cm3_cmd_group[cmd_group_bits], + cm3_cca_bits, 1 << mcp_bits, + cm3_tr[cm3_tr_bits], sched_bit); + } else { + buf[0] = 0; + } + + pr_err("CM_ERROR=%llx %s <%s>\n", cm_error, + cm3_causes[cause], buf); + pr_err("CM_ADDR =%llx\n", cm_addr); + pr_err("CM_OTHER=%llx %s\n", cm_other, cm3_causes[ocause]); + } + + /* reprime cause register */ + write_gcr_error_cause(cm_error); +} + +unsigned int mips_cps_first_online_in_cluster(int *first_cpu) +{ + unsigned int local_cl = cpu_cluster(¤t_cpu_data); + struct cpumask *local_cl_mask; + + /* + * mips_cps_cluster_bootcfg is allocated in cps_prepare_cpus. If it is + * not yet done, then we are so early that only one CPU is running, so + * it is the first online CPU in the cluster. + */ + if (IS_ENABLED(CONFIG_MIPS_CPS) && mips_cps_cluster_bootcfg) + local_cl_mask = &mips_cps_cluster_bootcfg[local_cl].cpumask; + else + return true; + + *first_cpu = cpumask_any_and_but(local_cl_mask, + cpu_online_mask, + smp_processor_id()); + return (*first_cpu >= nr_cpu_ids); +} diff --git a/arch/mips/kernel/mips-cpc.c b/arch/mips/kernel/mips-cpc.c new file mode 100644 index 000000000000..3e386f7e1545 --- /dev/null +++ b/arch/mips/kernel/mips-cpc.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2013 Imagination Technologies + * Author: Paul Burton <paul.burton@mips.com> + */ + +#include <linux/bitfield.h> +#include <linux/errno.h> +#include <linux/percpu.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/spinlock.h> + +#include <asm/mips-cps.h> + +void __iomem *mips_cpc_base; + +static DEFINE_PER_CPU_ALIGNED(spinlock_t, cpc_core_lock); + +static DEFINE_PER_CPU_ALIGNED(unsigned long, cpc_core_lock_flags); + +phys_addr_t __weak mips_cpc_default_phys_base(void) +{ + struct device_node *cpc_node; + struct resource res; + int err; + + cpc_node = of_find_compatible_node(of_root, NULL, "mti,mips-cpc"); + if (cpc_node) { + err = of_address_to_resource(cpc_node, 0, &res); + of_node_put(cpc_node); + if (!err) + return res.start; + } + + return 0; +} + +/** + * mips_cpc_phys_base - retrieve the physical base address of the CPC + * + * This function returns the physical base address of the Cluster Power + * Controller memory mapped registers, or 0 if no Cluster Power Controller + * is present. + */ +static phys_addr_t mips_cpc_phys_base(void) +{ + unsigned long cpc_base; + + if (!mips_cm_present()) + return 0; + + if (!(read_gcr_cpc_status() & CM_GCR_CPC_STATUS_EX)) + return 0; + + /* If the CPC is already enabled, leave it so */ + cpc_base = read_gcr_cpc_base(); + if (cpc_base & CM_GCR_CPC_BASE_CPCEN) + return cpc_base & CM_GCR_CPC_BASE_CPCBASE; + + /* Otherwise, use the default address */ + cpc_base = mips_cpc_default_phys_base(); + if (!cpc_base) + return cpc_base; + + /* Enable the CPC, mapped at the default address */ + write_gcr_cpc_base(cpc_base | CM_GCR_CPC_BASE_CPCEN); + return cpc_base; +} + +int mips_cpc_probe(void) +{ + phys_addr_t addr; + unsigned int cpu; + + for_each_possible_cpu(cpu) + spin_lock_init(&per_cpu(cpc_core_lock, cpu)); + + addr = mips_cpc_phys_base(); + if (!addr) + return -ENODEV; + + mips_cpc_base = ioremap(addr, 0x8000); + if (!mips_cpc_base) + return -ENXIO; + + return 0; +} + +void mips_cpc_lock_other(unsigned int core) +{ + unsigned int curr_core; + + if (mips_cm_revision() >= CM_REV_CM3) + /* Systems with CM >= 3 lock the CPC via mips_cm_lock_other */ + return; + + preempt_disable(); + curr_core = cpu_core(¤t_cpu_data); + spin_lock_irqsave(&per_cpu(cpc_core_lock, curr_core), + per_cpu(cpc_core_lock_flags, curr_core)); + write_cpc_cl_other(FIELD_PREP(CPC_Cx_OTHER_CORENUM, core)); + + /* + * Ensure the core-other region reflects the appropriate core & + * VP before any accesses to it occur. + */ + mb(); +} + +void mips_cpc_unlock_other(void) +{ + unsigned int curr_core; + + if (mips_cm_revision() >= CM_REV_CM3) + /* Systems with CM >= 3 lock the CPC via mips_cm_lock_other */ + return; + + curr_core = cpu_core(¤t_cpu_data); + spin_unlock_irqrestore(&per_cpu(cpc_core_lock, curr_core), + per_cpu(cpc_core_lock_flags, curr_core)); + preempt_enable(); +} diff --git a/arch/mips/kernel/mips-mt-fpaff.c b/arch/mips/kernel/mips-mt-fpaff.c index cb098628aee8..10172fc4f627 100644 --- a/arch/mips/kernel/mips-mt-fpaff.c +++ b/arch/mips/kernel/mips-mt-fpaff.c @@ -1,5 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 /* - * General MIPS MT support routines, usable in AP/SP, SMVP, or SMTC kernels + * General MIPS MT support routines, usable in AP/SP and SMVP. * Copyright (C) 2005 Mips Technologies, Inc */ #include <linux/cpu.h> @@ -9,9 +10,12 @@ #include <linux/kernel.h> #include <linux/init.h> #include <linux/sched.h> +#include <linux/sched/task.h> +#include <linux/cred.h> #include <linux/security.h> #include <linux/types.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> +#include <asm/syscalls.h> /* * CPU mask used to set process affinity for MT VPEs/TCs with FPUs @@ -73,13 +77,13 @@ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len, if (copy_from_user(&new_mask, user_mask_ptr, sizeof(new_mask))) return -EFAULT; - get_online_cpus(); + cpus_read_lock(); rcu_read_lock(); p = find_process_by_pid(pid); if (!p) { rcu_read_unlock(); - put_online_cpus(); + cpus_read_unlock(); return -ESRCH; } @@ -99,9 +103,10 @@ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len, retval = -ENOMEM; goto out_free_new_mask; } - retval = -EPERM; - if (!check_same_owner(p) && !capable(CAP_SYS_NICE)) + if (!check_same_owner(p) && !capable(CAP_SYS_NICE)) { + retval = -EPERM; goto out_unlock; + } retval = security_task_setscheduler(p); if (retval) @@ -114,8 +119,8 @@ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len, /* Compute new global allowed CPU set if necessary */ ti = task_thread_info(p); if (test_ti_thread_flag(ti, TIF_FPUBOUND) && - cpus_intersects(*new_mask, mt_fpu_cpumask)) { - cpus_and(*effective_mask, *new_mask, mt_fpu_cpumask); + cpumask_intersects(new_mask, &mt_fpu_cpumask)) { + cpumask_and(effective_mask, new_mask, &mt_fpu_cpumask); retval = set_cpus_allowed_ptr(p, effective_mask); } else { cpumask_copy(effective_mask, new_mask); @@ -143,7 +148,7 @@ out_free_cpus_allowed: free_cpumask_var(cpus_allowed); out_put_task: put_task_struct(p); - put_online_cpus(); + cpus_read_unlock(); return retval; } @@ -154,7 +159,7 @@ asmlinkage long mipsmt_sys_sched_getaffinity(pid_t pid, unsigned int len, unsigned long __user *user_mask_ptr) { unsigned int real_len; - cpumask_t mask; + cpumask_t allowed, mask; int retval; struct task_struct *p; @@ -162,8 +167,8 @@ asmlinkage long mipsmt_sys_sched_getaffinity(pid_t pid, unsigned int len, if (len < real_len) return -EINVAL; - get_online_cpus(); - read_lock(&tasklist_lock); + cpus_read_lock(); + rcu_read_lock(); retval = -ESRCH; p = find_process_by_pid(pid); @@ -173,11 +178,12 @@ asmlinkage long mipsmt_sys_sched_getaffinity(pid_t pid, unsigned int len, if (retval) goto out_unlock; - cpumask_and(&mask, &p->thread.user_cpus_allowed, cpu_possible_mask); + cpumask_or(&allowed, &p->thread.user_cpus_allowed, p->cpus_ptr); + cpumask_and(&mask, &allowed, cpu_active_mask); out_unlock: - read_unlock(&tasklist_lock); - put_online_cpus(); + rcu_read_unlock(); + cpus_read_unlock(); if (retval) return retval; if (copy_to_user(user_mask_ptr, &mask, real_len)) diff --git a/arch/mips/kernel/mips-mt.c b/arch/mips/kernel/mips-mt.c index 6ded9bd1489c..2ef610650a9e 100644 --- a/arch/mips/kernel/mips-mt.c +++ b/arch/mips/kernel/mips-mt.c @@ -1,5 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 /* - * General MIPS MT support routines, usable in AP/SP, SMVP, or SMTC kernels + * General MIPS MT support routines, usable in AP/SP and SMVP. * Copyright (C) 2005 Mips Technologies, Inc */ @@ -18,6 +19,7 @@ #include <asm/mipsmtregs.h> #include <asm/r4kcache.h> #include <asm/cacheflush.h> +#include <asm/mips_mt.h> int vpelimit; @@ -41,102 +43,11 @@ static int __init maxtcs(char *str) __setup("maxtcs=", maxtcs); -/* - * Dump new MIPS MT state for the core. Does not leave TCs halted. - * Takes an argument which taken to be a pre-call MVPControl value. - */ - -void mips_mt_regdump(unsigned long mvpctl) -{ - unsigned long flags; - unsigned long vpflags; - unsigned long mvpconf0; - int nvpe; - int ntc; - int i; - int tc; - unsigned long haltval; - unsigned long tcstatval; -#ifdef CONFIG_MIPS_MT_SMTC - void smtc_soft_dump(void); -#endif /* CONFIG_MIPT_MT_SMTC */ - - local_irq_save(flags); - vpflags = dvpe(); - printk("=== MIPS MT State Dump ===\n"); - printk("-- Global State --\n"); - printk(" MVPControl Passed: %08lx\n", mvpctl); - printk(" MVPControl Read: %08lx\n", vpflags); - printk(" MVPConf0 : %08lx\n", (mvpconf0 = read_c0_mvpconf0())); - nvpe = ((mvpconf0 & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; - ntc = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; - printk("-- per-VPE State --\n"); - for (i = 0; i < nvpe; i++) { - for (tc = 0; tc < ntc; tc++) { - settc(tc); - if ((read_tc_c0_tcbind() & TCBIND_CURVPE) == i) { - printk(" VPE %d\n", i); - printk(" VPEControl : %08lx\n", - read_vpe_c0_vpecontrol()); - printk(" VPEConf0 : %08lx\n", - read_vpe_c0_vpeconf0()); - printk(" VPE%d.Status : %08lx\n", - i, read_vpe_c0_status()); - printk(" VPE%d.EPC : %08lx %pS\n", - i, read_vpe_c0_epc(), - (void *) read_vpe_c0_epc()); - printk(" VPE%d.Cause : %08lx\n", - i, read_vpe_c0_cause()); - printk(" VPE%d.Config7 : %08lx\n", - i, read_vpe_c0_config7()); - break; /* Next VPE */ - } - } - } - printk("-- per-TC State --\n"); - for (tc = 0; tc < ntc; tc++) { - settc(tc); - if (read_tc_c0_tcbind() == read_c0_tcbind()) { - /* Are we dumping ourself? */ - haltval = 0; /* Then we're not halted, and mustn't be */ - tcstatval = flags; /* And pre-dump TCStatus is flags */ - printk(" TC %d (current TC with VPE EPC above)\n", tc); - } else { - haltval = read_tc_c0_tchalt(); - write_tc_c0_tchalt(1); - tcstatval = read_tc_c0_tcstatus(); - printk(" TC %d\n", tc); - } - printk(" TCStatus : %08lx\n", tcstatval); - printk(" TCBind : %08lx\n", read_tc_c0_tcbind()); - printk(" TCRestart : %08lx %pS\n", - read_tc_c0_tcrestart(), (void *) read_tc_c0_tcrestart()); - printk(" TCHalt : %08lx\n", haltval); - printk(" TCContext : %08lx\n", read_tc_c0_tccontext()); - if (!haltval) - write_tc_c0_tchalt(0); - } -#ifdef CONFIG_MIPS_MT_SMTC - smtc_soft_dump(); -#endif /* CONFIG_MIPT_MT_SMTC */ - printk("===========================\n"); - evpe(vpflags); - local_irq_restore(flags); -} - -static int mt_opt_norps; static int mt_opt_rpsctl = -1; static int mt_opt_nblsu = -1; static int mt_opt_forceconfig7; static int mt_opt_config7 = -1; -static int __init rps_disable(char *s) -{ - mt_opt_norps = 1; - return 1; -} -__setup("norps", rps_disable); - static int __init rpsctl_set(char *str) { get_option(&str, &mt_opt_rpsctl); @@ -159,40 +70,6 @@ static int __init config7_set(char *str) } __setup("config7=", config7_set); -/* Experimental cache flush control parameters that should go away some day */ -int mt_protiflush; -int mt_protdflush; -int mt_n_iflushes = 1; -int mt_n_dflushes = 1; - -static int __init set_protiflush(char *s) -{ - mt_protiflush = 1; - return 1; -} -__setup("protiflush", set_protiflush); - -static int __init set_protdflush(char *s) -{ - mt_protdflush = 1; - return 1; -} -__setup("protdflush", set_protdflush); - -static int __init niflush(char *s) -{ - get_option(&s, &mt_n_iflushes); - return 1; -} -__setup("niflush=", niflush); - -static int __init ndflush(char *s) -{ - get_option(&s, &mt_n_dflushes); - return 1; -} -__setup("ndflush=", ndflush); - static unsigned int itc_base; static int __init set_itc_base(char *str) @@ -208,9 +85,6 @@ void mips_mt_set_cpuoptions(void) unsigned int oconfig7 = read_c0_config7(); unsigned int nconfig7 = oconfig7; - if (mt_opt_norps) { - printk("\"norps\" option deprecated: use \"rpsctl=\"\n"); - } if (mt_opt_rpsctl >= 0) { printk("34K return prediction stack override set to %d.\n", mt_opt_rpsctl); @@ -237,16 +111,6 @@ void mips_mt_set_cpuoptions(void) printk("Config7: 0x%08x\n", read_c0_config7()); } - /* Report Cache management debug options */ - if (mt_protiflush) - printk("I-cache flushes single-threaded\n"); - if (mt_protdflush) - printk("D-cache flushes single-threaded\n"); - if (mt_n_iflushes != 1) - printk("I-Cache Flushes Repeated %d times\n", mt_n_iflushes); - if (mt_n_dflushes != 1) - printk("D-Cache Flushes Repeated %d times\n", mt_n_dflushes); - if (itc_base != 0) { /* * Configure ITC mapping. This code is very @@ -258,9 +122,8 @@ void mips_mt_set_cpuoptions(void) unsigned long ectlval; unsigned long itcblkgrn; - /* ErrCtl register is known as "ecc" to Linux */ - ectlval = read_c0_ecc(); - write_c0_ecc(ectlval | (0x1 << 26)); + ectlval = read_c0_errctl(); + write_c0_errctl(ectlval | (0x1 << 26)); ehb(); #define INDEX_0 (0x80000000) #define INDEX_8 (0x80000008) @@ -281,51 +144,20 @@ void mips_mt_set_cpuoptions(void) ehb(); /* Write out to ITU with CACHE op */ cache_op(Index_Store_Tag_D, INDEX_0); - write_c0_ecc(ectlval); + write_c0_errctl(ectlval); ehb(); printk("Mapped %ld ITC cells starting at 0x%08x\n", ((itcblkgrn & 0x7fe00000) >> 20), itc_base); } } -/* - * Function to protect cache flushes from concurrent execution - * depends on MP software model chosen. - */ +const struct class mt_class = { + .name = "mt", +}; -void mt_cflush_lockdown(void) +static int __init mips_mt_init(void) { -#ifdef CONFIG_MIPS_MT_SMTC - void smtc_cflush_lockdown(void); - - smtc_cflush_lockdown(); -#endif /* CONFIG_MIPS_MT_SMTC */ - /* FILL IN VSMP and AP/SP VERSIONS HERE */ -} - -void mt_cflush_release(void) -{ -#ifdef CONFIG_MIPS_MT_SMTC - void smtc_cflush_release(void); - - smtc_cflush_release(); -#endif /* CONFIG_MIPS_MT_SMTC */ - /* FILL IN VSMP and AP/SP VERSIONS HERE */ -} - -struct class *mt_class; - -static int __init mt_init(void) -{ - struct class *mtc; - - mtc = class_create(THIS_MODULE, "mt"); - if (IS_ERR(mtc)) - return PTR_ERR(mtc); - - mt_class = mtc; - - return 0; + return class_register(&mt_class); } -subsys_initcall(mt_init); +subsys_initcall(mips_mt_init); diff --git a/arch/mips/kernel/mips-r2-to-r6-emul.c b/arch/mips/kernel/mips-r2-to-r6-emul.c new file mode 100644 index 000000000000..750fe569862b --- /dev/null +++ b/arch/mips/kernel/mips-r2-to-r6-emul.c @@ -0,0 +1,2363 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (c) 2014 Imagination Technologies Ltd. + * Author: Leonid Yegoshin <Leonid.Yegoshin@imgtec.com> + * Author: Markos Chandras <markos.chandras@imgtec.com> + * + * MIPS R2 user space instruction emulator for MIPS R6 + * + */ +#include <linux/bug.h> +#include <linux/compiler.h> +#include <linux/debugfs.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/ptrace.h> +#include <linux/seq_file.h> + +#include <asm/asm.h> +#include <asm/branch.h> +#include <asm/break.h> +#include <asm/debug.h> +#include <asm/fpu.h> +#include <asm/fpu_emulator.h> +#include <asm/inst.h> +#include <asm/mips-r2-to-r6-emul.h> +#include <asm/local.h> +#include <asm/mipsregs.h> +#include <asm/ptrace.h> +#include <linux/uaccess.h> + +#ifdef CONFIG_64BIT +#define ADDIU "daddiu " +#define INS "dins " +#define EXT "dext " +#else +#define ADDIU "addiu " +#define INS "ins " +#define EXT "ext " +#endif /* CONFIG_64BIT */ + +#define SB "sb " +#define LB "lb " +#define LL "ll " +#define SC "sc " + +#ifdef CONFIG_DEBUG_FS +static DEFINE_PER_CPU(struct mips_r2_emulator_stats, mipsr2emustats); +static DEFINE_PER_CPU(struct mips_r2_emulator_stats, mipsr2bdemustats); +static DEFINE_PER_CPU(struct mips_r2br_emulator_stats, mipsr2bremustats); +#endif + +extern const unsigned int fpucondbit[8]; + +#define MIPS_R2_EMUL_TOTAL_PASS 10 + +int mipsr2_emulation = 0; + +static int __init mipsr2emu_enable(char *s) +{ + mipsr2_emulation = 1; + + pr_info("MIPS R2-to-R6 Emulator Enabled!"); + + return 1; +} +__setup("mipsr2emu", mipsr2emu_enable); + +/** + * mipsr6_emul - Emulate some frequent R2/R5/R6 instructions in delay slot + * for performance instead of the traditional way of using a stack trampoline + * which is rather slow. + * @regs: Process register set + * @ir: Instruction + */ +static inline int mipsr6_emul(struct pt_regs *regs, u32 ir) +{ + switch (MIPSInst_OPCODE(ir)) { + case addiu_op: + if (MIPSInst_RT(ir)) + regs->regs[MIPSInst_RT(ir)] = + (s32)regs->regs[MIPSInst_RS(ir)] + + (s32)MIPSInst_SIMM(ir); + return 0; + case daddiu_op: + if (IS_ENABLED(CONFIG_32BIT)) + break; + + if (MIPSInst_RT(ir)) + regs->regs[MIPSInst_RT(ir)] = + (s64)regs->regs[MIPSInst_RS(ir)] + + (s64)MIPSInst_SIMM(ir); + return 0; + case lwc1_op: + case swc1_op: + case cop1_op: + case cop1x_op: + /* FPU instructions in delay slot */ + return -SIGFPE; + case spec_op: + switch (MIPSInst_FUNC(ir)) { + case or_op: + if (MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = + regs->regs[MIPSInst_RS(ir)] | + regs->regs[MIPSInst_RT(ir)]; + return 0; + case sll_op: + if (MIPSInst_RS(ir)) + break; + + if (MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = + (s32)(((u32)regs->regs[MIPSInst_RT(ir)]) << + MIPSInst_FD(ir)); + return 0; + case srl_op: + if (MIPSInst_RS(ir)) + break; + + if (MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = + (s32)(((u32)regs->regs[MIPSInst_RT(ir)]) >> + MIPSInst_FD(ir)); + return 0; + case addu_op: + if (MIPSInst_FD(ir)) + break; + + if (MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = + (s32)((u32)regs->regs[MIPSInst_RS(ir)] + + (u32)regs->regs[MIPSInst_RT(ir)]); + return 0; + case subu_op: + if (MIPSInst_FD(ir)) + break; + + if (MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = + (s32)((u32)regs->regs[MIPSInst_RS(ir)] - + (u32)regs->regs[MIPSInst_RT(ir)]); + return 0; + case dsll_op: + if (IS_ENABLED(CONFIG_32BIT) || MIPSInst_RS(ir)) + break; + + if (MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = + (s64)(((u64)regs->regs[MIPSInst_RT(ir)]) << + MIPSInst_FD(ir)); + return 0; + case dsrl_op: + if (IS_ENABLED(CONFIG_32BIT) || MIPSInst_RS(ir)) + break; + + if (MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = + (s64)(((u64)regs->regs[MIPSInst_RT(ir)]) >> + MIPSInst_FD(ir)); + return 0; + case daddu_op: + if (IS_ENABLED(CONFIG_32BIT) || MIPSInst_FD(ir)) + break; + + if (MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = + (u64)regs->regs[MIPSInst_RS(ir)] + + (u64)regs->regs[MIPSInst_RT(ir)]; + return 0; + case dsubu_op: + if (IS_ENABLED(CONFIG_32BIT) || MIPSInst_FD(ir)) + break; + + if (MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = + (s64)((u64)regs->regs[MIPSInst_RS(ir)] - + (u64)regs->regs[MIPSInst_RT(ir)]); + return 0; + } + break; + default: + pr_debug("No fastpath BD emulation for instruction 0x%08x (op: %02x)\n", + ir, MIPSInst_OPCODE(ir)); + } + + return SIGILL; +} + +/** + * movf_func - Emulate a MOVF instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int movf_func(struct pt_regs *regs, u32 ir) +{ + u32 csr; + u32 cond; + + csr = current->thread.fpu.fcr31; + cond = fpucondbit[MIPSInst_RT(ir) >> 2]; + + if (((csr & cond) == 0) && MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = regs->regs[MIPSInst_RS(ir)]; + + MIPS_R2_STATS(movs); + + return 0; +} + +/** + * movt_func - Emulate a MOVT instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int movt_func(struct pt_regs *regs, u32 ir) +{ + u32 csr; + u32 cond; + + csr = current->thread.fpu.fcr31; + cond = fpucondbit[MIPSInst_RT(ir) >> 2]; + + if (((csr & cond) != 0) && MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = regs->regs[MIPSInst_RS(ir)]; + + MIPS_R2_STATS(movs); + + return 0; +} + +/** + * jr_func - Emulate a JR instruction. + * @pt_regs: Process register set + * @ir: Instruction + * + * Returns SIGILL if JR was in delay slot, SIGEMT if we + * can't compute the EPC, SIGSEGV if we can't access the + * userland instruction or 0 on success. + */ +static int jr_func(struct pt_regs *regs, u32 ir) +{ + int err; + unsigned long cepc, epc, nepc; + u32 nir; + + if (delay_slot(regs)) + return SIGILL; + + /* EPC after the RI/JR instruction */ + nepc = regs->cp0_epc; + /* Roll back to the reserved R2 JR instruction */ + regs->cp0_epc -= 4; + epc = regs->cp0_epc; + err = __compute_return_epc(regs); + + if (err < 0) + return SIGEMT; + + + /* Computed EPC */ + cepc = regs->cp0_epc; + + /* Get DS instruction */ + err = __get_user(nir, (u32 __user *)nepc); + if (err) + return SIGSEGV; + + MIPS_R2BR_STATS(jrs); + + /* If nir == 0(NOP), then nothing else to do */ + if (nir) { + /* + * Negative err means FPU instruction in BD-slot, + * Zero err means 'BD-slot emulation done' + * For anything else we go back to trampoline emulation. + */ + err = mipsr6_emul(regs, nir); + if (err > 0) { + regs->cp0_epc = nepc; + err = mips_dsemul(regs, nir, epc, cepc); + if (err == SIGILL) + err = SIGEMT; + MIPS_R2_STATS(dsemul); + } + } + + return err; +} + +/** + * movz_func - Emulate a MOVZ instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int movz_func(struct pt_regs *regs, u32 ir) +{ + if (((regs->regs[MIPSInst_RT(ir)]) == 0) && MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = regs->regs[MIPSInst_RS(ir)]; + MIPS_R2_STATS(movs); + + return 0; +} + +/** + * movn_func - Emulate a MOVZ instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int movn_func(struct pt_regs *regs, u32 ir) +{ + if (((regs->regs[MIPSInst_RT(ir)]) != 0) && MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = regs->regs[MIPSInst_RS(ir)]; + MIPS_R2_STATS(movs); + + return 0; +} + +/** + * mfhi_func - Emulate a MFHI instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int mfhi_func(struct pt_regs *regs, u32 ir) +{ + if (MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = regs->hi; + + MIPS_R2_STATS(hilo); + + return 0; +} + +/** + * mthi_func - Emulate a MTHI instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int mthi_func(struct pt_regs *regs, u32 ir) +{ + regs->hi = regs->regs[MIPSInst_RS(ir)]; + + MIPS_R2_STATS(hilo); + + return 0; +} + +/** + * mflo_func - Emulate a MFLO instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int mflo_func(struct pt_regs *regs, u32 ir) +{ + if (MIPSInst_RD(ir)) + regs->regs[MIPSInst_RD(ir)] = regs->lo; + + MIPS_R2_STATS(hilo); + + return 0; +} + +/** + * mtlo_func - Emulate a MTLO instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int mtlo_func(struct pt_regs *regs, u32 ir) +{ + regs->lo = regs->regs[MIPSInst_RS(ir)]; + + MIPS_R2_STATS(hilo); + + return 0; +} + +/** + * mult_func - Emulate a MULT instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int mult_func(struct pt_regs *regs, u32 ir) +{ + s64 res; + s32 rt, rs; + + rt = regs->regs[MIPSInst_RT(ir)]; + rs = regs->regs[MIPSInst_RS(ir)]; + res = (s64)rt * (s64)rs; + + rs = res; + regs->lo = (s64)rs; + rt = res >> 32; + res = (s64)rt; + regs->hi = res; + + MIPS_R2_STATS(muls); + + return 0; +} + +/** + * multu_func - Emulate a MULTU instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int multu_func(struct pt_regs *regs, u32 ir) +{ + u64 res; + u32 rt, rs; + + rt = regs->regs[MIPSInst_RT(ir)]; + rs = regs->regs[MIPSInst_RS(ir)]; + res = (u64)rt * (u64)rs; + rt = res; + regs->lo = (s64)(s32)rt; + regs->hi = (s64)(s32)(res >> 32); + + MIPS_R2_STATS(muls); + + return 0; +} + +/** + * div_func - Emulate a DIV instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int div_func(struct pt_regs *regs, u32 ir) +{ + s32 rt, rs; + + rt = regs->regs[MIPSInst_RT(ir)]; + rs = regs->regs[MIPSInst_RS(ir)]; + + regs->lo = (s64)(rs / rt); + regs->hi = (s64)(rs % rt); + + MIPS_R2_STATS(divs); + + return 0; +} + +/** + * divu_func - Emulate a DIVU instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int divu_func(struct pt_regs *regs, u32 ir) +{ + u32 rt, rs; + + rt = regs->regs[MIPSInst_RT(ir)]; + rs = regs->regs[MIPSInst_RS(ir)]; + + regs->lo = (s64)(rs / rt); + regs->hi = (s64)(rs % rt); + + MIPS_R2_STATS(divs); + + return 0; +} + +/** + * dmult_func - Emulate a DMULT instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 on success or SIGILL for 32-bit kernels. + */ +static int dmult_func(struct pt_regs *regs, u32 ir) +{ + s64 res; + s64 rt, rs; + + if (IS_ENABLED(CONFIG_32BIT)) + return SIGILL; + + rt = regs->regs[MIPSInst_RT(ir)]; + rs = regs->regs[MIPSInst_RS(ir)]; + res = rt * rs; + + regs->lo = res; + __asm__ __volatile__( + "dmuh %0, %1, %2\t\n" + : "=r"(res) + : "r"(rt), "r"(rs)); + + regs->hi = res; + + MIPS_R2_STATS(muls); + + return 0; +} + +/** + * dmultu_func - Emulate a DMULTU instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 on success or SIGILL for 32-bit kernels. + */ +static int dmultu_func(struct pt_regs *regs, u32 ir) +{ + u64 res; + u64 rt, rs; + + if (IS_ENABLED(CONFIG_32BIT)) + return SIGILL; + + rt = regs->regs[MIPSInst_RT(ir)]; + rs = regs->regs[MIPSInst_RS(ir)]; + res = rt * rs; + + regs->lo = res; + __asm__ __volatile__( + "dmuhu %0, %1, %2\t\n" + : "=r"(res) + : "r"(rt), "r"(rs)); + + regs->hi = res; + + MIPS_R2_STATS(muls); + + return 0; +} + +/** + * ddiv_func - Emulate a DDIV instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 on success or SIGILL for 32-bit kernels. + */ +static int ddiv_func(struct pt_regs *regs, u32 ir) +{ + s64 rt, rs; + + if (IS_ENABLED(CONFIG_32BIT)) + return SIGILL; + + rt = regs->regs[MIPSInst_RT(ir)]; + rs = regs->regs[MIPSInst_RS(ir)]; + + regs->lo = rs / rt; + regs->hi = rs % rt; + + MIPS_R2_STATS(divs); + + return 0; +} + +/** + * ddivu_func - Emulate a DDIVU instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 on success or SIGILL for 32-bit kernels. + */ +static int ddivu_func(struct pt_regs *regs, u32 ir) +{ + u64 rt, rs; + + if (IS_ENABLED(CONFIG_32BIT)) + return SIGILL; + + rt = regs->regs[MIPSInst_RT(ir)]; + rs = regs->regs[MIPSInst_RS(ir)]; + + regs->lo = rs / rt; + regs->hi = rs % rt; + + MIPS_R2_STATS(divs); + + return 0; +} + +/* R6 removed instructions for the SPECIAL opcode */ +static const struct r2_decoder_table spec_op_table[] = { + { 0xfc1ff83f, 0x00000008, jr_func }, + { 0xfc00ffff, 0x00000018, mult_func }, + { 0xfc00ffff, 0x00000019, multu_func }, + { 0xfc00ffff, 0x0000001c, dmult_func }, + { 0xfc00ffff, 0x0000001d, dmultu_func }, + { 0xffff07ff, 0x00000010, mfhi_func }, + { 0xfc1fffff, 0x00000011, mthi_func }, + { 0xffff07ff, 0x00000012, mflo_func }, + { 0xfc1fffff, 0x00000013, mtlo_func }, + { 0xfc0307ff, 0x00000001, movf_func }, + { 0xfc0307ff, 0x00010001, movt_func }, + { 0xfc0007ff, 0x0000000a, movz_func }, + { 0xfc0007ff, 0x0000000b, movn_func }, + { 0xfc00ffff, 0x0000001a, div_func }, + { 0xfc00ffff, 0x0000001b, divu_func }, + { 0xfc00ffff, 0x0000001e, ddiv_func }, + { 0xfc00ffff, 0x0000001f, ddivu_func }, + {} +}; + +/** + * madd_func - Emulate a MADD instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int madd_func(struct pt_regs *regs, u32 ir) +{ + s64 res; + s32 rt, rs; + + rt = regs->regs[MIPSInst_RT(ir)]; + rs = regs->regs[MIPSInst_RS(ir)]; + res = (s64)rt * (s64)rs; + rt = regs->hi; + rs = regs->lo; + res += ((((s64)rt) << 32) | (u32)rs); + + rt = res; + regs->lo = (s64)rt; + rs = res >> 32; + regs->hi = (s64)rs; + + MIPS_R2_STATS(dsps); + + return 0; +} + +/** + * maddu_func - Emulate a MADDU instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int maddu_func(struct pt_regs *regs, u32 ir) +{ + u64 res; + u32 rt, rs; + + rt = regs->regs[MIPSInst_RT(ir)]; + rs = regs->regs[MIPSInst_RS(ir)]; + res = (u64)rt * (u64)rs; + rt = regs->hi; + rs = regs->lo; + res += ((((s64)rt) << 32) | (u32)rs); + + rt = res; + regs->lo = (s64)(s32)rt; + rs = res >> 32; + regs->hi = (s64)(s32)rs; + + MIPS_R2_STATS(dsps); + + return 0; +} + +/** + * msub_func - Emulate a MSUB instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int msub_func(struct pt_regs *regs, u32 ir) +{ + s64 res; + s32 rt, rs; + + rt = regs->regs[MIPSInst_RT(ir)]; + rs = regs->regs[MIPSInst_RS(ir)]; + res = (s64)rt * (s64)rs; + rt = regs->hi; + rs = regs->lo; + res = ((((s64)rt) << 32) | (u32)rs) - res; + + rt = res; + regs->lo = (s64)rt; + rs = res >> 32; + regs->hi = (s64)rs; + + MIPS_R2_STATS(dsps); + + return 0; +} + +/** + * msubu_func - Emulate a MSUBU instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int msubu_func(struct pt_regs *regs, u32 ir) +{ + u64 res; + u32 rt, rs; + + rt = regs->regs[MIPSInst_RT(ir)]; + rs = regs->regs[MIPSInst_RS(ir)]; + res = (u64)rt * (u64)rs; + rt = regs->hi; + rs = regs->lo; + res = ((((s64)rt) << 32) | (u32)rs) - res; + + rt = res; + regs->lo = (s64)(s32)rt; + rs = res >> 32; + regs->hi = (s64)(s32)rs; + + MIPS_R2_STATS(dsps); + + return 0; +} + +/** + * mul_func - Emulate a MUL instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int mul_func(struct pt_regs *regs, u32 ir) +{ + s64 res; + s32 rt, rs; + + if (!MIPSInst_RD(ir)) + return 0; + rt = regs->regs[MIPSInst_RT(ir)]; + rs = regs->regs[MIPSInst_RS(ir)]; + res = (s64)rt * (s64)rs; + + rs = res; + regs->regs[MIPSInst_RD(ir)] = (s64)rs; + + MIPS_R2_STATS(muls); + + return 0; +} + +/** + * clz_func - Emulate a CLZ instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int clz_func(struct pt_regs *regs, u32 ir) +{ + u32 res; + u32 rs; + + if (!MIPSInst_RD(ir)) + return 0; + + rs = regs->regs[MIPSInst_RS(ir)]; + __asm__ __volatile__("clz %0, %1" : "=r"(res) : "r"(rs)); + regs->regs[MIPSInst_RD(ir)] = res; + + MIPS_R2_STATS(bops); + + return 0; +} + +/** + * clo_func - Emulate a CLO instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ + +static int clo_func(struct pt_regs *regs, u32 ir) +{ + u32 res; + u32 rs; + + if (!MIPSInst_RD(ir)) + return 0; + + rs = regs->regs[MIPSInst_RS(ir)]; + __asm__ __volatile__("clo %0, %1" : "=r"(res) : "r"(rs)); + regs->regs[MIPSInst_RD(ir)] = res; + + MIPS_R2_STATS(bops); + + return 0; +} + +/** + * dclz_func - Emulate a DCLZ instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int dclz_func(struct pt_regs *regs, u32 ir) +{ + u64 res; + u64 rs; + + if (IS_ENABLED(CONFIG_32BIT)) + return SIGILL; + + if (!MIPSInst_RD(ir)) + return 0; + + rs = regs->regs[MIPSInst_RS(ir)]; + __asm__ __volatile__("dclz %0, %1" : "=r"(res) : "r"(rs)); + regs->regs[MIPSInst_RD(ir)] = res; + + MIPS_R2_STATS(bops); + + return 0; +} + +/** + * dclo_func - Emulate a DCLO instruction + * @regs: Process register set + * @ir: Instruction + * + * Returns 0 since it always succeeds. + */ +static int dclo_func(struct pt_regs *regs, u32 ir) +{ + u64 res; + u64 rs; + + if (IS_ENABLED(CONFIG_32BIT)) + return SIGILL; + + if (!MIPSInst_RD(ir)) + return 0; + + rs = regs->regs[MIPSInst_RS(ir)]; + __asm__ __volatile__("dclo %0, %1" : "=r"(res) : "r"(rs)); + regs->regs[MIPSInst_RD(ir)] = res; + + MIPS_R2_STATS(bops); + + return 0; +} + +/* R6 removed instructions for the SPECIAL2 opcode */ +static const struct r2_decoder_table spec2_op_table[] = { + { 0xfc00ffff, 0x70000000, madd_func }, + { 0xfc00ffff, 0x70000001, maddu_func }, + { 0xfc0007ff, 0x70000002, mul_func }, + { 0xfc00ffff, 0x70000004, msub_func }, + { 0xfc00ffff, 0x70000005, msubu_func }, + { 0xfc0007ff, 0x70000020, clz_func }, + { 0xfc0007ff, 0x70000021, clo_func }, + { 0xfc0007ff, 0x70000024, dclz_func }, + { 0xfc0007ff, 0x70000025, dclo_func }, + { } +}; + +static inline int mipsr2_find_op_func(struct pt_regs *regs, u32 inst, + const struct r2_decoder_table *table) +{ + const struct r2_decoder_table *p; + int err; + + for (p = table; p->func; p++) { + if ((inst & p->mask) == p->code) { + err = (p->func)(regs, inst); + return err; + } + } + return SIGILL; +} + +/** + * mipsr2_decoder: Decode and emulate a MIPS R2 instruction + * @regs: Process register set + * @inst: Instruction to decode and emulate + * @fcr31: Floating Point Control and Status Register Cause bits returned + */ +int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) +{ + int err = 0; + unsigned long vaddr; + u32 nir; + unsigned long cpc, epc, nepc, r31, res, rs, rt; + + void __user *fault_addr = NULL; + int pass = 0; + +repeat: + r31 = regs->regs[31]; + epc = regs->cp0_epc; + err = compute_return_epc(regs); + if (err < 0) { + BUG(); + return SIGEMT; + } + pr_debug("Emulating the 0x%08x R2 instruction @ 0x%08lx (pass=%d))\n", + inst, epc, pass); + + switch (MIPSInst_OPCODE(inst)) { + case spec_op: + err = mipsr2_find_op_func(regs, inst, spec_op_table); + if (err < 0) { + /* FPU instruction under JR */ + regs->cp0_cause |= CAUSEF_BD; + goto fpu_emul; + } + break; + case spec2_op: + err = mipsr2_find_op_func(regs, inst, spec2_op_table); + break; + case bcond_op: + rt = MIPSInst_RT(inst); + rs = MIPSInst_RS(inst); + switch (rt) { + case tgei_op: + if ((long)regs->regs[rs] >= MIPSInst_SIMM(inst)) + do_trap_or_bp(regs, 0, 0, "TGEI"); + + MIPS_R2_STATS(traps); + + break; + case tgeiu_op: + if (regs->regs[rs] >= MIPSInst_UIMM(inst)) + do_trap_or_bp(regs, 0, 0, "TGEIU"); + + MIPS_R2_STATS(traps); + + break; + case tlti_op: + if ((long)regs->regs[rs] < MIPSInst_SIMM(inst)) + do_trap_or_bp(regs, 0, 0, "TLTI"); + + MIPS_R2_STATS(traps); + + break; + case tltiu_op: + if (regs->regs[rs] < MIPSInst_UIMM(inst)) + do_trap_or_bp(regs, 0, 0, "TLTIU"); + + MIPS_R2_STATS(traps); + + break; + case teqi_op: + if (regs->regs[rs] == MIPSInst_SIMM(inst)) + do_trap_or_bp(regs, 0, 0, "TEQI"); + + MIPS_R2_STATS(traps); + + break; + case tnei_op: + if (regs->regs[rs] != MIPSInst_SIMM(inst)) + do_trap_or_bp(regs, 0, 0, "TNEI"); + + MIPS_R2_STATS(traps); + + break; + case bltzl_op: + case bgezl_op: + case bltzall_op: + case bgezall_op: + if (delay_slot(regs)) { + err = SIGILL; + break; + } + regs->regs[31] = r31; + regs->cp0_epc = epc; + err = __compute_return_epc(regs); + if (err < 0) + return SIGEMT; + if (err != BRANCH_LIKELY_TAKEN) + break; + cpc = regs->cp0_epc; + nepc = epc + 4; + err = __get_user(nir, (u32 __user *)nepc); + if (err) { + err = SIGSEGV; + break; + } + /* + * This will probably be optimized away when + * CONFIG_DEBUG_FS is not enabled + */ + switch (rt) { + case bltzl_op: + MIPS_R2BR_STATS(bltzl); + break; + case bgezl_op: + MIPS_R2BR_STATS(bgezl); + break; + case bltzall_op: + MIPS_R2BR_STATS(bltzall); + break; + case bgezall_op: + MIPS_R2BR_STATS(bgezall); + break; + } + + switch (MIPSInst_OPCODE(nir)) { + case cop1_op: + case cop1x_op: + case lwc1_op: + case swc1_op: + regs->cp0_cause |= CAUSEF_BD; + goto fpu_emul; + } + if (nir) { + err = mipsr6_emul(regs, nir); + if (err > 0) { + err = mips_dsemul(regs, nir, epc, cpc); + if (err == SIGILL) + err = SIGEMT; + MIPS_R2_STATS(dsemul); + } + } + break; + case bltzal_op: + case bgezal_op: + if (delay_slot(regs)) { + err = SIGILL; + break; + } + regs->regs[31] = r31; + regs->cp0_epc = epc; + err = __compute_return_epc(regs); + if (err < 0) + return SIGEMT; + cpc = regs->cp0_epc; + nepc = epc + 4; + err = __get_user(nir, (u32 __user *)nepc); + if (err) { + err = SIGSEGV; + break; + } + /* + * This will probably be optimized away when + * CONFIG_DEBUG_FS is not enabled + */ + switch (rt) { + case bltzal_op: + MIPS_R2BR_STATS(bltzal); + break; + case bgezal_op: + MIPS_R2BR_STATS(bgezal); + break; + } + + switch (MIPSInst_OPCODE(nir)) { + case cop1_op: + case cop1x_op: + case lwc1_op: + case swc1_op: + regs->cp0_cause |= CAUSEF_BD; + goto fpu_emul; + } + if (nir) { + err = mipsr6_emul(regs, nir); + if (err > 0) { + err = mips_dsemul(regs, nir, epc, cpc); + if (err == SIGILL) + err = SIGEMT; + MIPS_R2_STATS(dsemul); + } + } + break; + default: + regs->regs[31] = r31; + regs->cp0_epc = epc; + err = SIGILL; + break; + } + break; + + case blezl_op: + case bgtzl_op: + /* + * For BLEZL and BGTZL, rt field must be set to 0. If this + * is not the case, this may be an encoding of a MIPS R6 + * instruction, so return to CPU execution if this occurs + */ + if (MIPSInst_RT(inst)) { + err = SIGILL; + break; + } + fallthrough; + case beql_op: + case bnel_op: + if (delay_slot(regs)) { + err = SIGILL; + break; + } + regs->regs[31] = r31; + regs->cp0_epc = epc; + err = __compute_return_epc(regs); + if (err < 0) + return SIGEMT; + if (err != BRANCH_LIKELY_TAKEN) + break; + cpc = regs->cp0_epc; + nepc = epc + 4; + err = __get_user(nir, (u32 __user *)nepc); + if (err) { + err = SIGSEGV; + break; + } + /* + * This will probably be optimized away when + * CONFIG_DEBUG_FS is not enabled + */ + switch (MIPSInst_OPCODE(inst)) { + case beql_op: + MIPS_R2BR_STATS(beql); + break; + case bnel_op: + MIPS_R2BR_STATS(bnel); + break; + case blezl_op: + MIPS_R2BR_STATS(blezl); + break; + case bgtzl_op: + MIPS_R2BR_STATS(bgtzl); + break; + } + + switch (MIPSInst_OPCODE(nir)) { + case cop1_op: + case cop1x_op: + case lwc1_op: + case swc1_op: + regs->cp0_cause |= CAUSEF_BD; + goto fpu_emul; + } + if (nir) { + err = mipsr6_emul(regs, nir); + if (err > 0) { + err = mips_dsemul(regs, nir, epc, cpc); + if (err == SIGILL) + err = SIGEMT; + MIPS_R2_STATS(dsemul); + } + } + break; + case lwc1_op: + case swc1_op: + case cop1_op: + case cop1x_op: +fpu_emul: + regs->regs[31] = r31; + regs->cp0_epc = epc; + + err = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 0, + &fault_addr); + + /* + * We can't allow the emulated instruction to leave any + * enabled Cause bits set in $fcr31. + */ + *fcr31 = res = mask_fcr31_x(current->thread.fpu.fcr31); + current->thread.fpu.fcr31 &= ~res; + + /* + * this is a tricky issue - lose_fpu() uses LL/SC atomics + * if FPU is owned and effectively cancels user level LL/SC. + * So, it could be logical to don't restore FPU ownership here. + * But the sequence of multiple FPU instructions is much much + * more often than LL-FPU-SC and I prefer loop here until + * next scheduler cycle cancels FPU ownership + */ + own_fpu(1); /* Restore FPU state. */ + + if (err) + current->thread.cp0_baduaddr = (unsigned long)fault_addr; + + MIPS_R2_STATS(fpus); + + break; + + case lwl_op: + rt = regs->regs[MIPSInst_RT(inst)]; + vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); + if (!access_ok((void __user *)vaddr, 4)) { + current->thread.cp0_baduaddr = vaddr; + err = SIGSEGV; + break; + } + __asm__ __volatile__( + " .set push\n" + " .set reorder\n" +#ifdef CONFIG_CPU_LITTLE_ENDIAN + "1:" LB "%1, 0(%2)\n" + INS "%0, %1, 24, 8\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + ADDIU "%2, %2, -1\n" + "2:" LB "%1, 0(%2)\n" + INS "%0, %1, 16, 8\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + ADDIU "%2, %2, -1\n" + "3:" LB "%1, 0(%2)\n" + INS "%0, %1, 8, 8\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + ADDIU "%2, %2, -1\n" + "4:" LB "%1, 0(%2)\n" + INS "%0, %1, 0, 8\n" +#else /* !CONFIG_CPU_LITTLE_ENDIAN */ + "1:" LB "%1, 0(%2)\n" + INS "%0, %1, 24, 8\n" + ADDIU "%2, %2, 1\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + "2:" LB "%1, 0(%2)\n" + INS "%0, %1, 16, 8\n" + ADDIU "%2, %2, 1\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + "3:" LB "%1, 0(%2)\n" + INS "%0, %1, 8, 8\n" + ADDIU "%2, %2, 1\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + "4:" LB "%1, 0(%2)\n" + INS "%0, %1, 0, 8\n" +#endif /* CONFIG_CPU_LITTLE_ENDIAN */ + "9: sll %0, %0, 0\n" + "10:\n" + " .insn\n" + " .section .fixup,\"ax\"\n" + "8: li %3,%4\n" + " j 10b\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + STR(PTR_WD) " 1b,8b\n" + STR(PTR_WD) " 2b,8b\n" + STR(PTR_WD) " 3b,8b\n" + STR(PTR_WD) " 4b,8b\n" + " .previous\n" + " .set pop\n" + : "+&r"(rt), "=&r"(rs), + "+&r"(vaddr), "+&r"(err) + : "i"(SIGSEGV)); + + if (MIPSInst_RT(inst) && !err) + regs->regs[MIPSInst_RT(inst)] = rt; + + MIPS_R2_STATS(loads); + + break; + + case lwr_op: + rt = regs->regs[MIPSInst_RT(inst)]; + vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); + if (!access_ok((void __user *)vaddr, 4)) { + current->thread.cp0_baduaddr = vaddr; + err = SIGSEGV; + break; + } + __asm__ __volatile__( + " .set push\n" + " .set reorder\n" +#ifdef CONFIG_CPU_LITTLE_ENDIAN + "1:" LB "%1, 0(%2)\n" + INS "%0, %1, 0, 8\n" + ADDIU "%2, %2, 1\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + "2:" LB "%1, 0(%2)\n" + INS "%0, %1, 8, 8\n" + ADDIU "%2, %2, 1\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + "3:" LB "%1, 0(%2)\n" + INS "%0, %1, 16, 8\n" + ADDIU "%2, %2, 1\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + "4:" LB "%1, 0(%2)\n" + INS "%0, %1, 24, 8\n" + " sll %0, %0, 0\n" +#else /* !CONFIG_CPU_LITTLE_ENDIAN */ + "1:" LB "%1, 0(%2)\n" + INS "%0, %1, 0, 8\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + ADDIU "%2, %2, -1\n" + "2:" LB "%1, 0(%2)\n" + INS "%0, %1, 8, 8\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + ADDIU "%2, %2, -1\n" + "3:" LB "%1, 0(%2)\n" + INS "%0, %1, 16, 8\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + ADDIU "%2, %2, -1\n" + "4:" LB "%1, 0(%2)\n" + INS "%0, %1, 24, 8\n" + " sll %0, %0, 0\n" +#endif /* CONFIG_CPU_LITTLE_ENDIAN */ + "9:\n" + "10:\n" + " .insn\n" + " .section .fixup,\"ax\"\n" + "8: li %3,%4\n" + " j 10b\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + STR(PTR_WD) " 1b,8b\n" + STR(PTR_WD) " 2b,8b\n" + STR(PTR_WD) " 3b,8b\n" + STR(PTR_WD) " 4b,8b\n" + " .previous\n" + " .set pop\n" + : "+&r"(rt), "=&r"(rs), + "+&r"(vaddr), "+&r"(err) + : "i"(SIGSEGV)); + if (MIPSInst_RT(inst) && !err) + regs->regs[MIPSInst_RT(inst)] = rt; + + MIPS_R2_STATS(loads); + + break; + + case swl_op: + rt = regs->regs[MIPSInst_RT(inst)]; + vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); + if (!access_ok((void __user *)vaddr, 4)) { + current->thread.cp0_baduaddr = vaddr; + err = SIGSEGV; + break; + } + __asm__ __volatile__( + " .set push\n" + " .set reorder\n" +#ifdef CONFIG_CPU_LITTLE_ENDIAN + EXT "%1, %0, 24, 8\n" + "1:" SB "%1, 0(%2)\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + ADDIU "%2, %2, -1\n" + EXT "%1, %0, 16, 8\n" + "2:" SB "%1, 0(%2)\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + ADDIU "%2, %2, -1\n" + EXT "%1, %0, 8, 8\n" + "3:" SB "%1, 0(%2)\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + ADDIU "%2, %2, -1\n" + EXT "%1, %0, 0, 8\n" + "4:" SB "%1, 0(%2)\n" +#else /* !CONFIG_CPU_LITTLE_ENDIAN */ + EXT "%1, %0, 24, 8\n" + "1:" SB "%1, 0(%2)\n" + ADDIU "%2, %2, 1\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + EXT "%1, %0, 16, 8\n" + "2:" SB "%1, 0(%2)\n" + ADDIU "%2, %2, 1\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + EXT "%1, %0, 8, 8\n" + "3:" SB "%1, 0(%2)\n" + ADDIU "%2, %2, 1\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + EXT "%1, %0, 0, 8\n" + "4:" SB "%1, 0(%2)\n" +#endif /* CONFIG_CPU_LITTLE_ENDIAN */ + "9:\n" + " .insn\n" + " .section .fixup,\"ax\"\n" + "8: li %3,%4\n" + " j 9b\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + STR(PTR_WD) " 1b,8b\n" + STR(PTR_WD) " 2b,8b\n" + STR(PTR_WD) " 3b,8b\n" + STR(PTR_WD) " 4b,8b\n" + " .previous\n" + " .set pop\n" + : "+&r"(rt), "=&r"(rs), + "+&r"(vaddr), "+&r"(err) + : "i"(SIGSEGV) + : "memory"); + + MIPS_R2_STATS(stores); + + break; + + case swr_op: + rt = regs->regs[MIPSInst_RT(inst)]; + vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); + if (!access_ok((void __user *)vaddr, 4)) { + current->thread.cp0_baduaddr = vaddr; + err = SIGSEGV; + break; + } + __asm__ __volatile__( + " .set push\n" + " .set reorder\n" +#ifdef CONFIG_CPU_LITTLE_ENDIAN + EXT "%1, %0, 0, 8\n" + "1:" SB "%1, 0(%2)\n" + ADDIU "%2, %2, 1\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + EXT "%1, %0, 8, 8\n" + "2:" SB "%1, 0(%2)\n" + ADDIU "%2, %2, 1\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + EXT "%1, %0, 16, 8\n" + "3:" SB "%1, 0(%2)\n" + ADDIU "%2, %2, 1\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + EXT "%1, %0, 24, 8\n" + "4:" SB "%1, 0(%2)\n" +#else /* !CONFIG_CPU_LITTLE_ENDIAN */ + EXT "%1, %0, 0, 8\n" + "1:" SB "%1, 0(%2)\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + ADDIU "%2, %2, -1\n" + EXT "%1, %0, 8, 8\n" + "2:" SB "%1, 0(%2)\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + ADDIU "%2, %2, -1\n" + EXT "%1, %0, 16, 8\n" + "3:" SB "%1, 0(%2)\n" + " andi %1, %2, 0x3\n" + " beq $0, %1, 9f\n" + ADDIU "%2, %2, -1\n" + EXT "%1, %0, 24, 8\n" + "4:" SB "%1, 0(%2)\n" +#endif /* CONFIG_CPU_LITTLE_ENDIAN */ + "9:\n" + " .insn\n" + " .section .fixup,\"ax\"\n" + "8: li %3,%4\n" + " j 9b\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + STR(PTR_WD) " 1b,8b\n" + STR(PTR_WD) " 2b,8b\n" + STR(PTR_WD) " 3b,8b\n" + STR(PTR_WD) " 4b,8b\n" + " .previous\n" + " .set pop\n" + : "+&r"(rt), "=&r"(rs), + "+&r"(vaddr), "+&r"(err) + : "i"(SIGSEGV) + : "memory"); + + MIPS_R2_STATS(stores); + + break; + + case ldl_op: + if (IS_ENABLED(CONFIG_32BIT)) { + err = SIGILL; + break; + } + + rt = regs->regs[MIPSInst_RT(inst)]; + vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); + if (!access_ok((void __user *)vaddr, 8)) { + current->thread.cp0_baduaddr = vaddr; + err = SIGSEGV; + break; + } + __asm__ __volatile__( + " .set push\n" + " .set reorder\n" +#ifdef CONFIG_CPU_LITTLE_ENDIAN + "1: lb %1, 0(%2)\n" + " dinsu %0, %1, 56, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "2: lb %1, 0(%2)\n" + " dinsu %0, %1, 48, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "3: lb %1, 0(%2)\n" + " dinsu %0, %1, 40, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "4: lb %1, 0(%2)\n" + " dinsu %0, %1, 32, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "5: lb %1, 0(%2)\n" + " dins %0, %1, 24, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "6: lb %1, 0(%2)\n" + " dins %0, %1, 16, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "7: lb %1, 0(%2)\n" + " dins %0, %1, 8, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "0: lb %1, 0(%2)\n" + " dins %0, %1, 0, 8\n" +#else /* !CONFIG_CPU_LITTLE_ENDIAN */ + "1: lb %1, 0(%2)\n" + " dinsu %0, %1, 56, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "2: lb %1, 0(%2)\n" + " dinsu %0, %1, 48, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "3: lb %1, 0(%2)\n" + " dinsu %0, %1, 40, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "4: lb %1, 0(%2)\n" + " dinsu %0, %1, 32, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "5: lb %1, 0(%2)\n" + " dins %0, %1, 24, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "6: lb %1, 0(%2)\n" + " dins %0, %1, 16, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "7: lb %1, 0(%2)\n" + " dins %0, %1, 8, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "0: lb %1, 0(%2)\n" + " dins %0, %1, 0, 8\n" +#endif /* CONFIG_CPU_LITTLE_ENDIAN */ + "9:\n" + " .insn\n" + " .section .fixup,\"ax\"\n" + "8: li %3,%4\n" + " j 9b\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + STR(PTR_WD) " 1b,8b\n" + STR(PTR_WD) " 2b,8b\n" + STR(PTR_WD) " 3b,8b\n" + STR(PTR_WD) " 4b,8b\n" + STR(PTR_WD) " 5b,8b\n" + STR(PTR_WD) " 6b,8b\n" + STR(PTR_WD) " 7b,8b\n" + STR(PTR_WD) " 0b,8b\n" + " .previous\n" + " .set pop\n" + : "+&r"(rt), "=&r"(rs), + "+&r"(vaddr), "+&r"(err) + : "i"(SIGSEGV)); + if (MIPSInst_RT(inst) && !err) + regs->regs[MIPSInst_RT(inst)] = rt; + + MIPS_R2_STATS(loads); + break; + + case ldr_op: + if (IS_ENABLED(CONFIG_32BIT)) { + err = SIGILL; + break; + } + + rt = regs->regs[MIPSInst_RT(inst)]; + vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); + if (!access_ok((void __user *)vaddr, 8)) { + current->thread.cp0_baduaddr = vaddr; + err = SIGSEGV; + break; + } + __asm__ __volatile__( + " .set push\n" + " .set reorder\n" +#ifdef CONFIG_CPU_LITTLE_ENDIAN + "1: lb %1, 0(%2)\n" + " dins %0, %1, 0, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "2: lb %1, 0(%2)\n" + " dins %0, %1, 8, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "3: lb %1, 0(%2)\n" + " dins %0, %1, 16, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "4: lb %1, 0(%2)\n" + " dins %0, %1, 24, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "5: lb %1, 0(%2)\n" + " dinsu %0, %1, 32, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "6: lb %1, 0(%2)\n" + " dinsu %0, %1, 40, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "7: lb %1, 0(%2)\n" + " dinsu %0, %1, 48, 8\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + "0: lb %1, 0(%2)\n" + " dinsu %0, %1, 56, 8\n" +#else /* !CONFIG_CPU_LITTLE_ENDIAN */ + "1: lb %1, 0(%2)\n" + " dins %0, %1, 0, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "2: lb %1, 0(%2)\n" + " dins %0, %1, 8, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "3: lb %1, 0(%2)\n" + " dins %0, %1, 16, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "4: lb %1, 0(%2)\n" + " dins %0, %1, 24, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "5: lb %1, 0(%2)\n" + " dinsu %0, %1, 32, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "6: lb %1, 0(%2)\n" + " dinsu %0, %1, 40, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "7: lb %1, 0(%2)\n" + " dinsu %0, %1, 48, 8\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + "0: lb %1, 0(%2)\n" + " dinsu %0, %1, 56, 8\n" +#endif /* CONFIG_CPU_LITTLE_ENDIAN */ + "9:\n" + " .insn\n" + " .section .fixup,\"ax\"\n" + "8: li %3,%4\n" + " j 9b\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + STR(PTR_WD) " 1b,8b\n" + STR(PTR_WD) " 2b,8b\n" + STR(PTR_WD) " 3b,8b\n" + STR(PTR_WD) " 4b,8b\n" + STR(PTR_WD) " 5b,8b\n" + STR(PTR_WD) " 6b,8b\n" + STR(PTR_WD) " 7b,8b\n" + STR(PTR_WD) " 0b,8b\n" + " .previous\n" + " .set pop\n" + : "+&r"(rt), "=&r"(rs), + "+&r"(vaddr), "+&r"(err) + : "i"(SIGSEGV)); + if (MIPSInst_RT(inst) && !err) + regs->regs[MIPSInst_RT(inst)] = rt; + + MIPS_R2_STATS(loads); + break; + + case sdl_op: + if (IS_ENABLED(CONFIG_32BIT)) { + err = SIGILL; + break; + } + + rt = regs->regs[MIPSInst_RT(inst)]; + vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); + if (!access_ok((void __user *)vaddr, 8)) { + current->thread.cp0_baduaddr = vaddr; + err = SIGSEGV; + break; + } + __asm__ __volatile__( + " .set push\n" + " .set reorder\n" +#ifdef CONFIG_CPU_LITTLE_ENDIAN + " dextu %1, %0, 56, 8\n" + "1: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dextu %1, %0, 48, 8\n" + "2: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dextu %1, %0, 40, 8\n" + "3: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dextu %1, %0, 32, 8\n" + "4: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dext %1, %0, 24, 8\n" + "5: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dext %1, %0, 16, 8\n" + "6: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dext %1, %0, 8, 8\n" + "7: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dext %1, %0, 0, 8\n" + "0: sb %1, 0(%2)\n" +#else /* !CONFIG_CPU_LITTLE_ENDIAN */ + " dextu %1, %0, 56, 8\n" + "1: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dextu %1, %0, 48, 8\n" + "2: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dextu %1, %0, 40, 8\n" + "3: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dextu %1, %0, 32, 8\n" + "4: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dext %1, %0, 24, 8\n" + "5: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dext %1, %0, 16, 8\n" + "6: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dext %1, %0, 8, 8\n" + "7: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dext %1, %0, 0, 8\n" + "0: sb %1, 0(%2)\n" +#endif /* CONFIG_CPU_LITTLE_ENDIAN */ + "9:\n" + " .insn\n" + " .section .fixup,\"ax\"\n" + "8: li %3,%4\n" + " j 9b\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + STR(PTR_WD) " 1b,8b\n" + STR(PTR_WD) " 2b,8b\n" + STR(PTR_WD) " 3b,8b\n" + STR(PTR_WD) " 4b,8b\n" + STR(PTR_WD) " 5b,8b\n" + STR(PTR_WD) " 6b,8b\n" + STR(PTR_WD) " 7b,8b\n" + STR(PTR_WD) " 0b,8b\n" + " .previous\n" + " .set pop\n" + : "+&r"(rt), "=&r"(rs), + "+&r"(vaddr), "+&r"(err) + : "i"(SIGSEGV) + : "memory"); + + MIPS_R2_STATS(stores); + break; + + case sdr_op: + if (IS_ENABLED(CONFIG_32BIT)) { + err = SIGILL; + break; + } + + rt = regs->regs[MIPSInst_RT(inst)]; + vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); + if (!access_ok((void __user *)vaddr, 8)) { + current->thread.cp0_baduaddr = vaddr; + err = SIGSEGV; + break; + } + __asm__ __volatile__( + " .set push\n" + " .set reorder\n" +#ifdef CONFIG_CPU_LITTLE_ENDIAN + " dext %1, %0, 0, 8\n" + "1: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dext %1, %0, 8, 8\n" + "2: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dext %1, %0, 16, 8\n" + "3: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dext %1, %0, 24, 8\n" + "4: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dextu %1, %0, 32, 8\n" + "5: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dextu %1, %0, 40, 8\n" + "6: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dextu %1, %0, 48, 8\n" + "7: sb %1, 0(%2)\n" + " daddiu %2, %2, 1\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " dextu %1, %0, 56, 8\n" + "0: sb %1, 0(%2)\n" +#else /* !CONFIG_CPU_LITTLE_ENDIAN */ + " dext %1, %0, 0, 8\n" + "1: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dext %1, %0, 8, 8\n" + "2: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dext %1, %0, 16, 8\n" + "3: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dext %1, %0, 24, 8\n" + "4: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dextu %1, %0, 32, 8\n" + "5: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dextu %1, %0, 40, 8\n" + "6: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dextu %1, %0, 48, 8\n" + "7: sb %1, 0(%2)\n" + " andi %1, %2, 0x7\n" + " beq $0, %1, 9f\n" + " daddiu %2, %2, -1\n" + " dextu %1, %0, 56, 8\n" + "0: sb %1, 0(%2)\n" +#endif /* CONFIG_CPU_LITTLE_ENDIAN */ + "9:\n" + " .insn\n" + " .section .fixup,\"ax\"\n" + "8: li %3,%4\n" + " j 9b\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + STR(PTR_WD) " 1b,8b\n" + STR(PTR_WD) " 2b,8b\n" + STR(PTR_WD) " 3b,8b\n" + STR(PTR_WD) " 4b,8b\n" + STR(PTR_WD) " 5b,8b\n" + STR(PTR_WD) " 6b,8b\n" + STR(PTR_WD) " 7b,8b\n" + STR(PTR_WD) " 0b,8b\n" + " .previous\n" + " .set pop\n" + : "+&r"(rt), "=&r"(rs), + "+&r"(vaddr), "+&r"(err) + : "i"(SIGSEGV) + : "memory"); + + MIPS_R2_STATS(stores); + + break; + case ll_op: + vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); + if (vaddr & 0x3) { + current->thread.cp0_baduaddr = vaddr; + err = SIGBUS; + break; + } + if (!access_ok((void __user *)vaddr, 4)) { + current->thread.cp0_baduaddr = vaddr; + err = SIGBUS; + break; + } + + if (!cpu_has_rw_llb) { + /* + * An LL/SC block can't be safely emulated without + * a Config5/LLB availability. So it's probably time to + * kill our process before things get any worse. This is + * because Config5/LLB allows us to use ERETNC so that + * the LLAddr/LLB bit is not cleared when we return from + * an exception. MIPS R2 LL/SC instructions trap with an + * RI exception so once we emulate them here, we return + * back to userland with ERETNC. That preserves the + * LLAddr/LLB so the subsequent SC instruction will + * succeed preserving the atomic semantics of the LL/SC + * block. Without that, there is no safe way to emulate + * an LL/SC block in MIPSR2 userland. + */ + pr_err("Can't emulate MIPSR2 LL/SC without Config5/LLB\n"); + err = SIGKILL; + break; + } + + __asm__ __volatile__( + "1:\n" + "ll %0, 0(%2)\n" + "2:\n" + ".insn\n" + ".section .fixup,\"ax\"\n" + "3:\n" + "li %1, %3\n" + "j 2b\n" + ".previous\n" + ".section __ex_table,\"a\"\n" + STR(PTR_WD) " 1b,3b\n" + ".previous\n" + : "=&r"(res), "+&r"(err) + : "r"(vaddr), "i"(SIGSEGV) + : "memory"); + + if (MIPSInst_RT(inst) && !err) + regs->regs[MIPSInst_RT(inst)] = res; + MIPS_R2_STATS(llsc); + + break; + + case sc_op: + vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); + if (vaddr & 0x3) { + current->thread.cp0_baduaddr = vaddr; + err = SIGBUS; + break; + } + if (!access_ok((void __user *)vaddr, 4)) { + current->thread.cp0_baduaddr = vaddr; + err = SIGBUS; + break; + } + + if (!cpu_has_rw_llb) { + /* + * An LL/SC block can't be safely emulated without + * a Config5/LLB availability. So it's probably time to + * kill our process before things get any worse. This is + * because Config5/LLB allows us to use ERETNC so that + * the LLAddr/LLB bit is not cleared when we return from + * an exception. MIPS R2 LL/SC instructions trap with an + * RI exception so once we emulate them here, we return + * back to userland with ERETNC. That preserves the + * LLAddr/LLB so the subsequent SC instruction will + * succeed preserving the atomic semantics of the LL/SC + * block. Without that, there is no safe way to emulate + * an LL/SC block in MIPSR2 userland. + */ + pr_err("Can't emulate MIPSR2 LL/SC without Config5/LLB\n"); + err = SIGKILL; + break; + } + + res = regs->regs[MIPSInst_RT(inst)]; + + __asm__ __volatile__( + "1:\n" + "sc %0, 0(%2)\n" + "2:\n" + ".insn\n" + ".section .fixup,\"ax\"\n" + "3:\n" + "li %1, %3\n" + "j 2b\n" + ".previous\n" + ".section __ex_table,\"a\"\n" + STR(PTR_WD) " 1b,3b\n" + ".previous\n" + : "+&r"(res), "+&r"(err) + : "r"(vaddr), "i"(SIGSEGV)); + + if (MIPSInst_RT(inst) && !err) + regs->regs[MIPSInst_RT(inst)] = res; + + MIPS_R2_STATS(llsc); + + break; + + case lld_op: + if (IS_ENABLED(CONFIG_32BIT)) { + err = SIGILL; + break; + } + + vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); + if (vaddr & 0x7) { + current->thread.cp0_baduaddr = vaddr; + err = SIGBUS; + break; + } + if (!access_ok((void __user *)vaddr, 8)) { + current->thread.cp0_baduaddr = vaddr; + err = SIGBUS; + break; + } + + if (!cpu_has_rw_llb) { + /* + * An LL/SC block can't be safely emulated without + * a Config5/LLB availability. So it's probably time to + * kill our process before things get any worse. This is + * because Config5/LLB allows us to use ERETNC so that + * the LLAddr/LLB bit is not cleared when we return from + * an exception. MIPS R2 LL/SC instructions trap with an + * RI exception so once we emulate them here, we return + * back to userland with ERETNC. That preserves the + * LLAddr/LLB so the subsequent SC instruction will + * succeed preserving the atomic semantics of the LL/SC + * block. Without that, there is no safe way to emulate + * an LL/SC block in MIPSR2 userland. + */ + pr_err("Can't emulate MIPSR2 LL/SC without Config5/LLB\n"); + err = SIGKILL; + break; + } + + __asm__ __volatile__( + "1:\n" + "lld %0, 0(%2)\n" + "2:\n" + ".insn\n" + ".section .fixup,\"ax\"\n" + "3:\n" + "li %1, %3\n" + "j 2b\n" + ".previous\n" + ".section __ex_table,\"a\"\n" + STR(PTR_WD) " 1b,3b\n" + ".previous\n" + : "=&r"(res), "+&r"(err) + : "r"(vaddr), "i"(SIGSEGV) + : "memory"); + if (MIPSInst_RT(inst) && !err) + regs->regs[MIPSInst_RT(inst)] = res; + + MIPS_R2_STATS(llsc); + + break; + + case scd_op: + if (IS_ENABLED(CONFIG_32BIT)) { + err = SIGILL; + break; + } + + vaddr = regs->regs[MIPSInst_RS(inst)] + MIPSInst_SIMM(inst); + if (vaddr & 0x7) { + current->thread.cp0_baduaddr = vaddr; + err = SIGBUS; + break; + } + if (!access_ok((void __user *)vaddr, 8)) { + current->thread.cp0_baduaddr = vaddr; + err = SIGBUS; + break; + } + + if (!cpu_has_rw_llb) { + /* + * An LL/SC block can't be safely emulated without + * a Config5/LLB availability. So it's probably time to + * kill our process before things get any worse. This is + * because Config5/LLB allows us to use ERETNC so that + * the LLAddr/LLB bit is not cleared when we return from + * an exception. MIPS R2 LL/SC instructions trap with an + * RI exception so once we emulate them here, we return + * back to userland with ERETNC. That preserves the + * LLAddr/LLB so the subsequent SC instruction will + * succeed preserving the atomic semantics of the LL/SC + * block. Without that, there is no safe way to emulate + * an LL/SC block in MIPSR2 userland. + */ + pr_err("Can't emulate MIPSR2 LL/SC without Config5/LLB\n"); + err = SIGKILL; + break; + } + + res = regs->regs[MIPSInst_RT(inst)]; + + __asm__ __volatile__( + "1:\n" + "scd %0, 0(%2)\n" + "2:\n" + ".insn\n" + ".section .fixup,\"ax\"\n" + "3:\n" + "li %1, %3\n" + "j 2b\n" + ".previous\n" + ".section __ex_table,\"a\"\n" + STR(PTR_WD) " 1b,3b\n" + ".previous\n" + : "+&r"(res), "+&r"(err) + : "r"(vaddr), "i"(SIGSEGV)); + + if (MIPSInst_RT(inst) && !err) + regs->regs[MIPSInst_RT(inst)] = res; + + MIPS_R2_STATS(llsc); + + break; + case pref_op: + /* skip it */ + break; + default: + err = SIGILL; + } + + /* + * Let's not return to userland just yet. It's costly and + * it's likely we have more R2 instructions to emulate + */ + if (!err && (pass++ < MIPS_R2_EMUL_TOTAL_PASS)) { + regs->cp0_cause &= ~CAUSEF_BD; + err = get_user(inst, (u32 __user *)regs->cp0_epc); + if (!err) + goto repeat; + + if (err < 0) + err = SIGSEGV; + } + + if (err && (err != SIGEMT)) { + regs->regs[31] = r31; + regs->cp0_epc = epc; + } + + /* Likely a MIPS R6 compatible instruction */ + if (pass && (err == SIGILL)) + err = 0; + + return err; +} + +#ifdef CONFIG_DEBUG_FS + +static int mipsr2_emul_show(struct seq_file *s, void *unused) +{ + + seq_printf(s, "Instruction\tTotal\tBDslot\n------------------------------\n"); + seq_printf(s, "movs\t\t%ld\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2emustats.movs), + (unsigned long)__this_cpu_read(mipsr2bdemustats.movs)); + seq_printf(s, "hilo\t\t%ld\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2emustats.hilo), + (unsigned long)__this_cpu_read(mipsr2bdemustats.hilo)); + seq_printf(s, "muls\t\t%ld\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2emustats.muls), + (unsigned long)__this_cpu_read(mipsr2bdemustats.muls)); + seq_printf(s, "divs\t\t%ld\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2emustats.divs), + (unsigned long)__this_cpu_read(mipsr2bdemustats.divs)); + seq_printf(s, "dsps\t\t%ld\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2emustats.dsps), + (unsigned long)__this_cpu_read(mipsr2bdemustats.dsps)); + seq_printf(s, "bops\t\t%ld\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2emustats.bops), + (unsigned long)__this_cpu_read(mipsr2bdemustats.bops)); + seq_printf(s, "traps\t\t%ld\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2emustats.traps), + (unsigned long)__this_cpu_read(mipsr2bdemustats.traps)); + seq_printf(s, "fpus\t\t%ld\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2emustats.fpus), + (unsigned long)__this_cpu_read(mipsr2bdemustats.fpus)); + seq_printf(s, "loads\t\t%ld\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2emustats.loads), + (unsigned long)__this_cpu_read(mipsr2bdemustats.loads)); + seq_printf(s, "stores\t\t%ld\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2emustats.stores), + (unsigned long)__this_cpu_read(mipsr2bdemustats.stores)); + seq_printf(s, "llsc\t\t%ld\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2emustats.llsc), + (unsigned long)__this_cpu_read(mipsr2bdemustats.llsc)); + seq_printf(s, "dsemul\t\t%ld\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2emustats.dsemul), + (unsigned long)__this_cpu_read(mipsr2bdemustats.dsemul)); + seq_printf(s, "jr\t\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2bremustats.jrs)); + seq_printf(s, "bltzl\t\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2bremustats.bltzl)); + seq_printf(s, "bgezl\t\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2bremustats.bgezl)); + seq_printf(s, "bltzll\t\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2bremustats.bltzll)); + seq_printf(s, "bgezll\t\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2bremustats.bgezll)); + seq_printf(s, "bltzal\t\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2bremustats.bltzal)); + seq_printf(s, "bgezal\t\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2bremustats.bgezal)); + seq_printf(s, "beql\t\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2bremustats.beql)); + seq_printf(s, "bnel\t\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2bremustats.bnel)); + seq_printf(s, "blezl\t\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2bremustats.blezl)); + seq_printf(s, "bgtzl\t\t%ld\n", + (unsigned long)__this_cpu_read(mipsr2bremustats.bgtzl)); + + return 0; +} + +static int mipsr2_clear_show(struct seq_file *s, void *unused) +{ + mipsr2_emul_show(s, unused); + + __this_cpu_write((mipsr2emustats).movs, 0); + __this_cpu_write((mipsr2bdemustats).movs, 0); + __this_cpu_write((mipsr2emustats).hilo, 0); + __this_cpu_write((mipsr2bdemustats).hilo, 0); + __this_cpu_write((mipsr2emustats).muls, 0); + __this_cpu_write((mipsr2bdemustats).muls, 0); + __this_cpu_write((mipsr2emustats).divs, 0); + __this_cpu_write((mipsr2bdemustats).divs, 0); + __this_cpu_write((mipsr2emustats).dsps, 0); + __this_cpu_write((mipsr2bdemustats).dsps, 0); + __this_cpu_write((mipsr2emustats).bops, 0); + __this_cpu_write((mipsr2bdemustats).bops, 0); + __this_cpu_write((mipsr2emustats).traps, 0); + __this_cpu_write((mipsr2bdemustats).traps, 0); + __this_cpu_write((mipsr2emustats).fpus, 0); + __this_cpu_write((mipsr2bdemustats).fpus, 0); + __this_cpu_write((mipsr2emustats).loads, 0); + __this_cpu_write((mipsr2bdemustats).loads, 0); + __this_cpu_write((mipsr2emustats).stores, 0); + __this_cpu_write((mipsr2bdemustats).stores, 0); + __this_cpu_write((mipsr2emustats).llsc, 0); + __this_cpu_write((mipsr2bdemustats).llsc, 0); + __this_cpu_write((mipsr2emustats).dsemul, 0); + __this_cpu_write((mipsr2bdemustats).dsemul, 0); + __this_cpu_write((mipsr2bremustats).jrs, 0); + __this_cpu_write((mipsr2bremustats).bltzl, 0); + __this_cpu_write((mipsr2bremustats).bgezl, 0); + __this_cpu_write((mipsr2bremustats).bltzll, 0); + __this_cpu_write((mipsr2bremustats).bgezll, 0); + __this_cpu_write((mipsr2bremustats).bltzall, 0); + __this_cpu_write((mipsr2bremustats).bgezall, 0); + __this_cpu_write((mipsr2bremustats).bltzal, 0); + __this_cpu_write((mipsr2bremustats).bgezal, 0); + __this_cpu_write((mipsr2bremustats).beql, 0); + __this_cpu_write((mipsr2bremustats).bnel, 0); + __this_cpu_write((mipsr2bremustats).blezl, 0); + __this_cpu_write((mipsr2bremustats).bgtzl, 0); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(mipsr2_emul); +DEFINE_SHOW_ATTRIBUTE(mipsr2_clear); + +static int __init mipsr2_init_debugfs(void) +{ + debugfs_create_file("r2_emul_stats", S_IRUGO, mips_debugfs_dir, NULL, + &mipsr2_emul_fops); + debugfs_create_file("r2_emul_stats_clear", S_IRUGO, mips_debugfs_dir, + NULL, &mipsr2_clear_fops); + return 0; +} + +device_initcall(mipsr2_init_debugfs); + +#endif /* CONFIG_DEBUG_FS */ diff --git a/arch/mips/kernel/mips_ksyms.c b/arch/mips/kernel/mips_ksyms.c deleted file mode 100644 index 6e58e97fcd39..000000000000 --- a/arch/mips/kernel/mips_ksyms.c +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Export MIPS-specific functions needed for loadable modules. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 1996, 97, 98, 99, 2000, 01, 03, 04, 05, 12 by Ralf Baechle - * Copyright (C) 1999, 2000, 01 Silicon Graphics, Inc. - */ -#include <linux/interrupt.h> -#include <linux/export.h> -#include <asm/checksum.h> -#include <linux/mm.h> -#include <asm/uaccess.h> -#include <asm/ftrace.h> - -extern void *__bzero(void *__s, size_t __count); -extern long __strncpy_from_user_nocheck_asm(char *__to, - const char *__from, long __len); -extern long __strncpy_from_user_asm(char *__to, const char *__from, - long __len); -extern long __strlen_user_nocheck_asm(const char *s); -extern long __strlen_user_asm(const char *s); -extern long __strnlen_user_nocheck_asm(const char *s); -extern long __strnlen_user_asm(const char *s); - -/* - * String functions - */ -EXPORT_SYMBOL(memset); -EXPORT_SYMBOL(memcpy); -EXPORT_SYMBOL(memmove); - -/* - * Functions that operate on entire pages. Mostly used by memory management. - */ -EXPORT_SYMBOL(clear_page); -EXPORT_SYMBOL(copy_page); - -/* - * Userspace access stuff. - */ -EXPORT_SYMBOL(__copy_user); -EXPORT_SYMBOL(__copy_user_inatomic); -EXPORT_SYMBOL(__bzero); -EXPORT_SYMBOL(__strncpy_from_user_nocheck_asm); -EXPORT_SYMBOL(__strncpy_from_user_asm); -EXPORT_SYMBOL(__strlen_user_nocheck_asm); -EXPORT_SYMBOL(__strlen_user_asm); -EXPORT_SYMBOL(__strnlen_user_nocheck_asm); -EXPORT_SYMBOL(__strnlen_user_asm); - -EXPORT_SYMBOL(csum_partial); -EXPORT_SYMBOL(csum_partial_copy_nocheck); -EXPORT_SYMBOL(__csum_partial_copy_user); - -EXPORT_SYMBOL(invalid_pte_table); -#ifdef CONFIG_FUNCTION_TRACER -/* _mcount is defined in arch/mips/kernel/mcount.S */ -EXPORT_SYMBOL(_mcount); -#endif diff --git a/arch/mips/kernel/mips_machine.c b/arch/mips/kernel/mips_machine.c deleted file mode 100644 index 876097529697..000000000000 --- a/arch/mips/kernel/mips_machine.c +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2008-2010 Gabor Juhos <juhosg@openwrt.org> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - */ -#include <linux/mm.h> -#include <linux/string.h> -#include <linux/slab.h> - -#include <asm/mips_machine.h> -#include <asm/prom.h> - -static struct mips_machine *mips_machine __initdata; - -#define for_each_machine(mach) \ - for ((mach) = (struct mips_machine *)&__mips_machines_start; \ - (mach) && \ - (unsigned long)(mach) < (unsigned long)&__mips_machines_end; \ - (mach)++) - -__init int mips_machtype_setup(char *id) -{ - struct mips_machine *mach; - - for_each_machine(mach) { - if (mach->mach_id == NULL) - continue; - - if (strcmp(mach->mach_id, id) == 0) { - mips_machtype = mach->mach_type; - return 0; - } - } - - pr_err("MIPS: no machine found for id '%s', supported machines:\n", id); - pr_err("%-24s %s\n", "id", "name"); - for_each_machine(mach) - pr_err("%-24s %s\n", mach->mach_id, mach->mach_name); - - return 1; -} - -__setup("machtype=", mips_machtype_setup); - -__init void mips_machine_setup(void) -{ - struct mips_machine *mach; - - for_each_machine(mach) { - if (mips_machtype == mach->mach_type) { - mips_machine = mach; - break; - } - } - - if (!mips_machine) - return; - - mips_set_machine_name(mips_machine->mach_name); - - if (mips_machine->mach_setup) - mips_machine->mach_setup(); -} diff --git a/arch/mips/kernel/module-rela.c b/arch/mips/kernel/module-rela.c deleted file mode 100644 index 2b70723071c3..000000000000 --- a/arch/mips/kernel/module-rela.c +++ /dev/null @@ -1,145 +0,0 @@ -/* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * Copyright (C) 2001 Rusty Russell. - * Copyright (C) 2003, 2004 Ralf Baechle (ralf@linux-mips.org) - * Copyright (C) 2005 Thiemo Seufer - */ - -#include <linux/elf.h> -#include <linux/err.h> -#include <linux/errno.h> -#include <linux/moduleloader.h> - -extern int apply_r_mips_none(struct module *me, u32 *location, Elf_Addr v); - -static int apply_r_mips_32_rela(struct module *me, u32 *location, Elf_Addr v) -{ - *location = v; - - return 0; -} - -static int apply_r_mips_26_rela(struct module *me, u32 *location, Elf_Addr v) -{ - if (v % 4) { - pr_err("module %s: dangerous R_MIPS_26 RELArelocation\n", - me->name); - return -ENOEXEC; - } - - if ((v & 0xf0000000) != (((unsigned long)location + 4) & 0xf0000000)) { - printk(KERN_ERR - "module %s: relocation overflow\n", - me->name); - return -ENOEXEC; - } - - *location = (*location & ~0x03ffffff) | ((v >> 2) & 0x03ffffff); - - return 0; -} - -static int apply_r_mips_hi16_rela(struct module *me, u32 *location, Elf_Addr v) -{ - *location = (*location & 0xffff0000) | - ((((long long) v + 0x8000LL) >> 16) & 0xffff); - - return 0; -} - -static int apply_r_mips_lo16_rela(struct module *me, u32 *location, Elf_Addr v) -{ - *location = (*location & 0xffff0000) | (v & 0xffff); - - return 0; -} - -static int apply_r_mips_64_rela(struct module *me, u32 *location, Elf_Addr v) -{ - *(Elf_Addr *)location = v; - - return 0; -} - -static int apply_r_mips_higher_rela(struct module *me, u32 *location, - Elf_Addr v) -{ - *location = (*location & 0xffff0000) | - ((((long long) v + 0x80008000LL) >> 32) & 0xffff); - - return 0; -} - -static int apply_r_mips_highest_rela(struct module *me, u32 *location, - Elf_Addr v) -{ - *location = (*location & 0xffff0000) | - ((((long long) v + 0x800080008000LL) >> 48) & 0xffff); - - return 0; -} - -static int (*reloc_handlers_rela[]) (struct module *me, u32 *location, - Elf_Addr v) = { - [R_MIPS_NONE] = apply_r_mips_none, - [R_MIPS_32] = apply_r_mips_32_rela, - [R_MIPS_26] = apply_r_mips_26_rela, - [R_MIPS_HI16] = apply_r_mips_hi16_rela, - [R_MIPS_LO16] = apply_r_mips_lo16_rela, - [R_MIPS_64] = apply_r_mips_64_rela, - [R_MIPS_HIGHER] = apply_r_mips_higher_rela, - [R_MIPS_HIGHEST] = apply_r_mips_highest_rela -}; - -int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, - unsigned int symindex, unsigned int relsec, - struct module *me) -{ - Elf_Mips_Rela *rel = (void *) sechdrs[relsec].sh_addr; - Elf_Sym *sym; - u32 *location; - unsigned int i; - Elf_Addr v; - int res; - - pr_debug("Applying relocate section %u to %u\n", relsec, - sechdrs[relsec].sh_info); - - for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { - /* This is where to make the change */ - location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr - + rel[i].r_offset; - /* This is the symbol it is referring to */ - sym = (Elf_Sym *)sechdrs[symindex].sh_addr - + ELF_MIPS_R_SYM(rel[i]); - if (IS_ERR_VALUE(sym->st_value)) { - /* Ignore unresolved weak symbol */ - if (ELF_ST_BIND(sym->st_info) == STB_WEAK) - continue; - printk(KERN_WARNING "%s: Unknown symbol %s\n", - me->name, strtab + sym->st_name); - return -ENOENT; - } - - v = sym->st_value + rel[i].r_addend; - - res = reloc_handlers_rela[ELF_MIPS_R_TYPE(rel[i])](me, location, v); - if (res) - return res; - } - - return 0; -} diff --git a/arch/mips/kernel/module.c b/arch/mips/kernel/module.c index 977a623d9253..ba0f62d8eff5 100644 --- a/arch/mips/kernel/module.c +++ b/arch/mips/kernel/module.c @@ -1,17 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * * Copyright (C) 2001 Rusty Russell. * Copyright (C) 2003, 2004 Ralf Baechle (ralf@linux-mips.org) @@ -20,18 +8,18 @@ #undef DEBUG +#include <linux/extable.h> #include <linux/moduleloader.h> #include <linux/elf.h> #include <linux/mm.h> -#include <linux/vmalloc.h> +#include <linux/numa.h> #include <linux/slab.h> #include <linux/fs.h> #include <linux/string.h> #include <linux/kernel.h> #include <linux/spinlock.h> #include <linux/jump_label.h> - -#include <asm/pgtable.h> /* MODULE_START */ +#include <asm/jump_label.h> struct mips_hi16 { struct mips_hi16 *next; @@ -42,52 +30,43 @@ struct mips_hi16 { static LIST_HEAD(dbe_list); static DEFINE_SPINLOCK(dbe_lock); -#ifdef MODULE_START -void *module_alloc(unsigned long size) +static void apply_r_mips_32(u32 *location, u32 base, Elf_Addr v) { - return __vmalloc_node_range(size, 1, MODULE_START, MODULE_END, - GFP_KERNEL, PAGE_KERNEL, -1, - __builtin_return_address(0)); + *location = base + v; } -#endif -int apply_r_mips_none(struct module *me, u32 *location, Elf_Addr v) -{ - return 0; -} - -static int apply_r_mips_32_rel(struct module *me, u32 *location, Elf_Addr v) -{ - *location += v; - - return 0; -} - -static int apply_r_mips_26_rel(struct module *me, u32 *location, Elf_Addr v) +static int apply_r_mips_26(struct module *me, u32 *location, u32 base, + Elf_Addr v) { if (v % 4) { - pr_err("module %s: dangerous R_MIPS_26 REL relocation\n", + pr_err("module %s: dangerous R_MIPS_26 relocation\n", me->name); return -ENOEXEC; } if ((v & 0xf0000000) != (((unsigned long)location + 4) & 0xf0000000)) { - printk(KERN_ERR - "module %s: relocation overflow\n", + pr_err("module %s: relocation overflow\n", me->name); return -ENOEXEC; } *location = (*location & ~0x03ffffff) | - ((*location + (v >> 2)) & 0x03ffffff); + ((base + (v >> 2)) & 0x03ffffff); return 0; } -static int apply_r_mips_hi16_rel(struct module *me, u32 *location, Elf_Addr v) +static int apply_r_mips_hi16(struct module *me, u32 *location, Elf_Addr v, + bool rela) { struct mips_hi16 *n; + if (rela) { + *location = (*location & 0xffff0000) | + ((((long long) v + 0x8000LL) >> 16) & 0xffff); + return 0; + } + /* * We cannot relocate this one now because we don't know the value of * the carry we need to add. Save the information, and let LO16 do the @@ -116,12 +95,18 @@ static void free_relocation_chain(struct mips_hi16 *l) } } -static int apply_r_mips_lo16_rel(struct module *me, u32 *location, Elf_Addr v) +static int apply_r_mips_lo16(struct module *me, u32 *location, + u32 base, Elf_Addr v, bool rela) { - unsigned long insnlo = *location; + unsigned long insnlo = base; struct mips_hi16 *l; Elf_Addr val, vallo; + if (rela) { + *location = (*location & 0xffff0000) | (v & 0xffff); + return 0; + } + /* Sign extend the addend we extract from the lo insn. */ vallo = ((insnlo & 0xffff) ^ 0x8000) - 0x8000; @@ -177,73 +162,232 @@ out_danger: free_relocation_chain(l); me->arch.r_mips_hi16_list = NULL; - pr_err("module %s: dangerous R_MIPS_LO16 REL relocation\n", me->name); + pr_err("module %s: dangerous R_MIPS_LO16 relocation\n", me->name); return -ENOEXEC; } -static int (*reloc_handlers_rel[]) (struct module *me, u32 *location, - Elf_Addr v) = { - [R_MIPS_NONE] = apply_r_mips_none, - [R_MIPS_32] = apply_r_mips_32_rel, - [R_MIPS_26] = apply_r_mips_26_rel, - [R_MIPS_HI16] = apply_r_mips_hi16_rel, - [R_MIPS_LO16] = apply_r_mips_lo16_rel -}; +static int apply_r_mips_pc(struct module *me, u32 *location, u32 base, + Elf_Addr v, unsigned int bits) +{ + unsigned long mask = GENMASK(bits - 1, 0); + unsigned long se_bits; + long offset; -int apply_relocate(Elf_Shdr *sechdrs, const char *strtab, - unsigned int symindex, unsigned int relsec, - struct module *me) + if (v % 4) { + pr_err("module %s: dangerous R_MIPS_PC%u relocation\n", + me->name, bits); + return -ENOEXEC; + } + + /* retrieve & sign extend implicit addend if any */ + offset = base & mask; + offset |= (offset & BIT(bits - 1)) ? ~mask : 0; + + offset += ((long)v - (long)location) >> 2; + + /* check the sign bit onwards are identical - ie. we didn't overflow */ + se_bits = (offset & BIT(bits - 1)) ? ~0ul : 0; + if ((offset & ~mask) != (se_bits & ~mask)) { + pr_err("module %s: relocation overflow\n", me->name); + return -ENOEXEC; + } + + *location = (*location & ~mask) | (offset & mask); + + return 0; +} + +static int apply_r_mips_pc16(struct module *me, u32 *location, u32 base, + Elf_Addr v) +{ + return apply_r_mips_pc(me, location, base, v, 16); +} + +static int apply_r_mips_pc21(struct module *me, u32 *location, u32 base, + Elf_Addr v) +{ + return apply_r_mips_pc(me, location, base, v, 21); +} + +static int apply_r_mips_pc26(struct module *me, u32 *location, u32 base, + Elf_Addr v) +{ + return apply_r_mips_pc(me, location, base, v, 26); +} + +static int apply_r_mips_64(u32 *location, Elf_Addr v, bool rela) +{ + if (WARN_ON(!rela)) + return -EINVAL; + + *(Elf_Addr *)location = v; + + return 0; +} + +static int apply_r_mips_higher(u32 *location, Elf_Addr v, bool rela) +{ + if (WARN_ON(!rela)) + return -EINVAL; + + *location = (*location & 0xffff0000) | + ((((long long)v + 0x80008000LL) >> 32) & 0xffff); + + return 0; +} + +static int apply_r_mips_highest(u32 *location, Elf_Addr v, bool rela) +{ + if (WARN_ON(!rela)) + return -EINVAL; + + *location = (*location & 0xffff0000) | + ((((long long)v + 0x800080008000LL) >> 48) & 0xffff); + + return 0; +} + +/** + * reloc_handler() - Apply a particular relocation to a module + * @type: type of the relocation to apply + * @me: the module to apply the reloc to + * @location: the address at which the reloc is to be applied + * @base: the existing value at location for REL-style; 0 for RELA-style + * @v: the value of the reloc, with addend for RELA-style + * @rela: indication of is this a RELA (true) or REL (false) relocation + * + * Each implemented relocation function applies a particular type of + * relocation to the module @me. Relocs that may be found in either REL or RELA + * variants can be handled by making use of the @base & @v parameters which are + * set to values which abstract the difference away from the particular reloc + * implementations. + * + * Return: 0 upon success, else -ERRNO + */ +static int reloc_handler(u32 type, struct module *me, u32 *location, u32 base, + Elf_Addr v, bool rela) { - Elf_Mips_Rel *rel = (void *) sechdrs[relsec].sh_addr; + switch (type) { + case R_MIPS_NONE: + break; + case R_MIPS_32: + apply_r_mips_32(location, base, v); + break; + case R_MIPS_26: + return apply_r_mips_26(me, location, base, v); + case R_MIPS_HI16: + return apply_r_mips_hi16(me, location, v, rela); + case R_MIPS_LO16: + return apply_r_mips_lo16(me, location, base, v, rela); + case R_MIPS_PC16: + return apply_r_mips_pc16(me, location, base, v); + case R_MIPS_PC21_S2: + return apply_r_mips_pc21(me, location, base, v); + case R_MIPS_PC26_S2: + return apply_r_mips_pc26(me, location, base, v); + case R_MIPS_64: + return apply_r_mips_64(location, v, rela); + case R_MIPS_HIGHER: + return apply_r_mips_higher(location, v, rela); + case R_MIPS_HIGHEST: + return apply_r_mips_highest(location, v, rela); + default: + pr_err("%s: Unknown relocation type %u\n", me->name, type); + return -EINVAL; + } + + return 0; +} + +static int __apply_relocate(Elf_Shdr *sechdrs, const char *strtab, + unsigned int symindex, unsigned int relsec, + struct module *me, bool rela) +{ + union { + Elf_Mips_Rel *rel; + Elf_Mips_Rela *rela; + } r; Elf_Sym *sym; - u32 *location; - unsigned int i; + u32 *location, base; + unsigned int i, type; Elf_Addr v; - int res; + int err = 0; + size_t reloc_sz; pr_debug("Applying relocate section %u to %u\n", relsec, sechdrs[relsec].sh_info); + r.rel = (void *)sechdrs[relsec].sh_addr; + reloc_sz = rela ? sizeof(*r.rela) : sizeof(*r.rel); me->arch.r_mips_hi16_list = NULL; - for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { + for (i = 0; i < sechdrs[relsec].sh_size / reloc_sz; i++) { /* This is where to make the change */ location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr - + rel[i].r_offset; + + r.rel->r_offset; /* This is the symbol it is referring to */ sym = (Elf_Sym *)sechdrs[symindex].sh_addr - + ELF_MIPS_R_SYM(rel[i]); - if (IS_ERR_VALUE(sym->st_value)) { + + ELF_MIPS_R_SYM(*r.rel); + if (sym->st_value >= -MAX_ERRNO) { /* Ignore unresolved weak symbol */ if (ELF_ST_BIND(sym->st_info) == STB_WEAK) continue; - printk(KERN_WARNING "%s: Unknown symbol %s\n", - me->name, strtab + sym->st_name); - return -ENOENT; + pr_warn("%s: Unknown symbol %s\n", + me->name, strtab + sym->st_name); + err = -ENOENT; + goto out; } - v = sym->st_value; + type = ELF_MIPS_R_TYPE(*r.rel); + + if (rela) { + v = sym->st_value + r.rela->r_addend; + base = 0; + r.rela = &r.rela[1]; + } else { + v = sym->st_value; + base = *location; + r.rel = &r.rel[1]; + } - res = reloc_handlers_rel[ELF_MIPS_R_TYPE(rel[i])](me, location, v); - if (res) - return res; + err = reloc_handler(type, me, location, base, v, rela); + if (err) + goto out; } +out: /* - * Normally the hi16 list should be deallocated at this point. A + * Normally the hi16 list should be deallocated at this point. A * malformed binary however could contain a series of R_MIPS_HI16 - * relocations not followed by a R_MIPS_LO16 relocation. In that - * case, free up the list and return an error. + * relocations not followed by a R_MIPS_LO16 relocation, or if we hit + * an error processing a reloc we might have gotten here before + * reaching the R_MIPS_LO16. In either case, free up the list and + * return an error. */ if (me->arch.r_mips_hi16_list) { free_relocation_chain(me->arch.r_mips_hi16_list); me->arch.r_mips_hi16_list = NULL; - - return -ENOEXEC; + err = err ?: -ENOEXEC; } - return 0; + return err; +} + +int apply_relocate(Elf_Shdr *sechdrs, const char *strtab, + unsigned int symindex, unsigned int relsec, + struct module *me) +{ + return __apply_relocate(sechdrs, strtab, symindex, relsec, me, false); +} + +#ifdef CONFIG_MODULES_USE_ELF_RELA +int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, + unsigned int symindex, unsigned int relsec, + struct module *me) +{ + return __apply_relocate(sechdrs, strtab, symindex, relsec, me, true); } +#endif /* CONFIG_MODULES_USE_ELF_RELA */ /* Given an address, look for it in the module exception tables. */ const struct exception_table_entry *search_module_dbetables(unsigned long addr) @@ -254,7 +398,8 @@ const struct exception_table_entry *search_module_dbetables(unsigned long addr) spin_lock_irqsave(&dbe_lock, flags); list_for_each_entry(dbe, &dbe_list, dbe_list) { - e = search_extable(dbe->dbe_start, dbe->dbe_end - 1, addr); + e = search_extable(dbe->dbe_start, + dbe->dbe_end - dbe->dbe_start, addr); if (e) break; } @@ -273,8 +418,8 @@ int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *s; char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; - /* Make jump label nops. */ - jump_label_apply_nops(me); + if (IS_ENABLED(CONFIG_JUMP_LABEL)) + jump_label_apply_nops(me); INIT_LIST_HEAD(&me->arch.dbe_list); for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { diff --git a/arch/mips/kernel/octeon_switch.S b/arch/mips/kernel/octeon_switch.S index 4204d76af854..447a3ea14aa1 100644 --- a/arch/mips/kernel/octeon_switch.S +++ b/arch/mips/kernel/octeon_switch.S @@ -11,26 +11,14 @@ * written by Carsten Langgaard, carstenl@mips.com */ #include <asm/asm.h> -#include <asm/cachectl.h> -#include <asm/fpregdef.h> -#include <asm/mipsregs.h> #include <asm/asm-offsets.h> -#include <asm/pgtable-bits.h> +#include <asm/mipsregs.h> #include <asm/regdef.h> #include <asm/stackframe.h> -#include <asm/thread_info.h> - -#include <asm/asmmacro.h> - -/* - * Offset to the current process status flags, the first 32 bytes of the - * stack are not used. - */ -#define ST_OFF (_THREAD_SIZE - 32 - PT_SIZE + PT_STATUS) /* * task_struct *resume(task_struct *prev, task_struct *next, - * struct thread_info *next_ti, int usedfpu) + * struct thread_info *next_ti) */ .align 7 LEAF(resume) @@ -42,7 +30,7 @@ #if CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0 /* Check if we need to store CVMSEG state */ - mfc0 t0, $11,7 /* CvmMemCtl */ + dmfc0 t0, $11,7 /* CvmMemCtl */ bbit0 t0, 6, 3f /* Is user access enabled? */ /* Store the CVMSEG state */ @@ -66,14 +54,14 @@ .set reorder /* Disable access to CVMSEG */ - mfc0 t0, $11,7 /* CvmMemCtl */ + dmfc0 t0, $11,7 /* CvmMemCtl */ xori t0, t0, 0x40 /* Bit 6 is CVMSEG user enable */ - mtc0 t0, $11,7 /* CvmMemCtl */ + dmtc0 t0, $11,7 /* CvmMemCtl */ #endif 3: -#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP) - PTR_L t8, __stack_chk_guard +#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) + PTR_LA t8, __stack_chk_guard LONG_L t9, TASK_STACK_CANARY(a1) LONG_S t9, 0(t8) #endif @@ -85,12 +73,7 @@ move $28, a2 cpu_restore_nonscratch a1 -#if (_THREAD_SIZE - 32) < 0x8000 - PTR_ADDIU t0, $28, _THREAD_SIZE - 32 -#else - PTR_LI t0, _THREAD_SIZE - 32 - PTR_ADDU t0, $28 -#endif + PTR_ADDU t0, $28, _THREAD_SIZE - 32 set_saved_sp t0, t1, t2 mfc0 t1, CP0_STATUS /* Do we really need this? */ @@ -109,6 +92,8 @@ * void octeon_cop2_save(struct octeon_cop2_state *a0) */ .align 7 + .set push + .set noreorder LEAF(octeon_cop2_save) dmfc0 t9, $9,7 /* CvmCtl register. */ @@ -119,17 +104,17 @@ dmfc2 t2, 0x0200 sd t0, OCTEON_CP2_CRC_IV(a0) sd t1, OCTEON_CP2_CRC_LENGTH(a0) - sd t2, OCTEON_CP2_CRC_POLY(a0) /* Skip next instructions if CvmCtl[NODFA_CP2] set */ bbit1 t9, 28, 1f + sd t2, OCTEON_CP2_CRC_POLY(a0) /* Save the LLM state */ dmfc2 t0, 0x0402 dmfc2 t1, 0x040A sd t0, OCTEON_CP2_LLM_DAT(a0) - sd t1, OCTEON_CP2_LLM_DAT+8(a0) 1: bbit1 t9, 26, 3f /* done if CvmCtl[NOCRYPTO] set */ + sd t1, OCTEON_CP2_LLM_DAT+8(a0) /* Save the COP2 crypto state */ /* this part is mostly common to both pass 1 and later revisions */ @@ -160,18 +145,20 @@ sd t2, OCTEON_CP2_AES_KEY+16(a0) dmfc2 t2, 0x0101 sd t3, OCTEON_CP2_AES_KEY+24(a0) - mfc0 t3, $15,0 /* Get the processor ID register */ + mfc0 v0, $15,0 /* Get the processor ID register */ sd t0, OCTEON_CP2_AES_KEYLEN(a0) - li t0, 0x000d0000 /* This is the processor ID of Octeon Pass1 */ + li v1, 0x000d0000 /* This is the processor ID of Octeon Pass1 */ sd t1, OCTEON_CP2_AES_RESULT(a0) - sd t2, OCTEON_CP2_AES_RESULT+8(a0) /* Skip to the Pass1 version of the remainder of the COP2 state */ - beq t3, t0, 2f + beq v0, v1, 2f + sd t2, OCTEON_CP2_AES_RESULT+8(a0) /* the non-pass1 state when !CvmCtl[NOCRYPTO] */ dmfc2 t1, 0x0240 dmfc2 t2, 0x0241 + ori v1, v1, 0x9500 /* lowest OCTEON III PrId*/ dmfc2 t3, 0x0242 + subu v1, v0, v1 /* prid - lowest OCTEON III PrId */ dmfc2 t0, 0x0243 sd t1, OCTEON_CP2_HSH_DATW(a0) dmfc2 t1, 0x0244 @@ -224,8 +211,16 @@ sd t1, OCTEON_CP2_GFM_MULT+8(a0) sd t2, OCTEON_CP2_GFM_POLY(a0) sd t3, OCTEON_CP2_GFM_RESULT(a0) - sd t0, OCTEON_CP2_GFM_RESULT+8(a0) + bltz v1, 4f + sd t0, OCTEON_CP2_GFM_RESULT+8(a0) + /* OCTEON III things*/ + dmfc2 t0, 0x024F + dmfc2 t1, 0x0050 + sd t0, OCTEON_CP2_SHA3(a0) + sd t1, OCTEON_CP2_SHA3+8(a0) +4: jr ra + nop 2: /* pass 1 special stuff when !CvmCtl[NOCRYPTO] */ dmfc2 t3, 0x0040 @@ -251,7 +246,9 @@ 3: /* pass 1 or CvmCtl[NOCRYPTO] set */ jr ra + nop END(octeon_cop2_save) + .set pop /* * void octeon_cop2_restore(struct octeon_cop2_state *a0) @@ -316,9 +313,9 @@ ld t2, OCTEON_CP2_AES_RESULT+8(a0) mfc0 t3, $15,0 /* Get the processor ID register */ dmtc2 t0, 0x0110 - li t0, 0x000d0000 /* This is the processor ID of Octeon Pass1 */ + li v0, 0x000d0000 /* This is the processor ID of Octeon Pass1 */ dmtc2 t1, 0x0100 - bne t0, t3, 3f /* Skip the next stuff for non-pass1 */ + bne v0, t3, 3f /* Skip the next stuff for non-pass1 */ dmtc2 t2, 0x0101 /* this code is specific for pass 1 */ @@ -346,6 +343,7 @@ 3: /* this is post-pass1 code */ ld t2, OCTEON_CP2_HSH_DATW(a0) + ori v0, v0, 0x9500 /* lowest OCTEON III PrId*/ ld t0, OCTEON_CP2_HSH_DATW+8(a0) ld t1, OCTEON_CP2_HSH_DATW+16(a0) dmtc2 t2, 0x0240 @@ -399,9 +397,15 @@ dmtc2 t2, 0x0259 ld t2, OCTEON_CP2_GFM_RESULT+8(a0) dmtc2 t0, 0x025E + subu v0, t3, v0 /* prid - lowest OCTEON III PrId */ dmtc2 t1, 0x025A - dmtc2 t2, 0x025B - + bltz v0, done_restore + dmtc2 t2, 0x025B + /* OCTEON III things*/ + ld t0, OCTEON_CP2_SHA3(a0) + ld t1, OCTEON_CP2_SHA3+8(a0) + dmtc2 t0, 0x0051 + dmtc2 t1, 0x0050 done_restore: jr ra nop @@ -412,18 +416,22 @@ done_restore: * void octeon_mult_save() * sp is assumed to point to a struct pt_regs * - * NOTE: This is called in SAVE_SOME in stackframe.h. It can only - * safely modify k0 and k1. + * NOTE: This is called in SAVE_TEMP in stackframe.h. It can + * safely modify v1,k0, k1,$10-$15, and $24. It will + * be overwritten with a processor specific version of the code. */ - .align 7 + .p2align 7 .set push .set noreorder LEAF(octeon_mult_save) - dmfc0 k0, $9,7 /* CvmCtl register. */ - bbit1 k0, 27, 1f /* Skip CvmCtl[NOMUL] */ + jr ra nop + .space 30 * 4, 0 + EXPORT(octeon_mult_save_end) + END(octeon_mult_save) - /* Save the multiplier state */ + LEAF(octeon_mult_save2) + /* Save the multiplier state OCTEON II and earlier*/ v3mulu k0, $0, $0 v3mulu k1, $0, $0 sd k0, PT_MTP(sp) /* PT_MTP has P0 */ @@ -438,44 +446,102 @@ done_restore: sd k0, PT_MPL+8(sp) /* PT_MPL+8 has MPL1 */ jr ra sd k1, PT_MPL+16(sp) /* PT_MPL+16 has MPL2 */ - -1: /* Resume here if CvmCtl[NOMUL] */ + EXPORT(octeon_mult_save2_end) + END(octeon_mult_save2) + + LEAF(octeon_mult_save3) + /* Save the multiplier state OCTEON III */ + v3mulu $10, $0, $0 /* read P0 */ + v3mulu $11, $0, $0 /* read P1 */ + v3mulu $12, $0, $0 /* read P2 */ + sd $10, PT_MTP+(0*8)(sp) /* store P0 */ + v3mulu $10, $0, $0 /* read P3 */ + sd $11, PT_MTP+(1*8)(sp) /* store P1 */ + v3mulu $11, $0, $0 /* read P4 */ + sd $12, PT_MTP+(2*8)(sp) /* store P2 */ + ori $13, $0, 1 + v3mulu $12, $0, $0 /* read P5 */ + sd $10, PT_MTP+(3*8)(sp) /* store P3 */ + v3mulu $13, $13, $0 /* P4-P0 = MPL5-MPL1, $13 = MPL0 */ + sd $11, PT_MTP+(4*8)(sp) /* store P4 */ + v3mulu $10, $0, $0 /* read MPL1 */ + sd $12, PT_MTP+(5*8)(sp) /* store P5 */ + v3mulu $11, $0, $0 /* read MPL2 */ + sd $13, PT_MPL+(0*8)(sp) /* store MPL0 */ + v3mulu $12, $0, $0 /* read MPL3 */ + sd $10, PT_MPL+(1*8)(sp) /* store MPL1 */ + v3mulu $10, $0, $0 /* read MPL4 */ + sd $11, PT_MPL+(2*8)(sp) /* store MPL2 */ + v3mulu $11, $0, $0 /* read MPL5 */ + sd $12, PT_MPL+(3*8)(sp) /* store MPL3 */ + sd $10, PT_MPL+(4*8)(sp) /* store MPL4 */ jr ra - END(octeon_mult_save) + sd $11, PT_MPL+(5*8)(sp) /* store MPL5 */ + EXPORT(octeon_mult_save3_end) + END(octeon_mult_save3) .set pop /* * void octeon_mult_restore() * sp is assumed to point to a struct pt_regs * - * NOTE: This is called in RESTORE_SOME in stackframe.h. + * NOTE: This is called in RESTORE_TEMP in stackframe.h. */ - .align 7 + .p2align 7 .set push .set noreorder LEAF(octeon_mult_restore) - dmfc0 k1, $9,7 /* CvmCtl register. */ - ld v0, PT_MPL(sp) /* MPL0 */ - ld v1, PT_MPL+8(sp) /* MPL1 */ - ld k0, PT_MPL+16(sp) /* MPL2 */ - bbit1 k1, 27, 1f /* Skip CvmCtl[NOMUL] */ - /* Normally falls through, so no time wasted here */ - nop + jr ra + nop + .space 30 * 4, 0 + EXPORT(octeon_mult_restore_end) + END(octeon_mult_restore) + LEAF(octeon_mult_restore2) + ld v0, PT_MPL(sp) /* MPL0 */ + ld v1, PT_MPL+8(sp) /* MPL1 */ + ld k0, PT_MPL+16(sp) /* MPL2 */ /* Restore the multiplier state */ - ld k1, PT_MTP+16(sp) /* P2 */ - MTM0 v0 /* MPL0 */ + ld k1, PT_MTP+16(sp) /* P2 */ + mtm0 v0 /* MPL0 */ ld v0, PT_MTP+8(sp) /* P1 */ - MTM1 v1 /* MPL1 */ - ld v1, PT_MTP(sp) /* P0 */ - MTM2 k0 /* MPL2 */ - MTP2 k1 /* P2 */ - MTP1 v0 /* P1 */ + mtm1 v1 /* MPL1 */ + ld v1, PT_MTP(sp) /* P0 */ + mtm2 k0 /* MPL2 */ + mtp2 k1 /* P2 */ + mtp1 v0 /* P1 */ jr ra - MTP0 v1 /* P0 */ - -1: /* Resume here if CvmCtl[NOMUL] */ + mtp0 v1 /* P0 */ + EXPORT(octeon_mult_restore2_end) + END(octeon_mult_restore2) + + LEAF(octeon_mult_restore3) + ld $12, PT_MPL+(0*8)(sp) /* read MPL0 */ + ld $13, PT_MPL+(3*8)(sp) /* read MPL3 */ + ld $10, PT_MPL+(1*8)(sp) /* read MPL1 */ + ld $11, PT_MPL+(4*8)(sp) /* read MPL4 */ + .word 0x718d0008 + /* mtm0 $12, $13 restore MPL0 and MPL3 */ + ld $12, PT_MPL+(2*8)(sp) /* read MPL2 */ + .word 0x714b000c + /* mtm1 $10, $11 restore MPL1 and MPL4 */ + ld $13, PT_MPL+(5*8)(sp) /* read MPL5 */ + ld $10, PT_MTP+(0*8)(sp) /* read P0 */ + ld $11, PT_MTP+(3*8)(sp) /* read P3 */ + .word 0x718d000d + /* mtm2 $12, $13 restore MPL2 and MPL5 */ + ld $12, PT_MTP+(1*8)(sp) /* read P1 */ + .word 0x714b0009 + /* mtp0 $10, $11 restore P0 and P3 */ + ld $13, PT_MTP+(4*8)(sp) /* read P4 */ + ld $10, PT_MTP+(2*8)(sp) /* read P2 */ + ld $11, PT_MTP+(5*8)(sp) /* read P5 */ + .word 0x718d000a + /* mtp1 $12, $13 restore P1 and P4 */ jr ra - nop - END(octeon_mult_restore) + .word 0x714b000b + /* mtp2 $10, $11 restore P2 and P5 */ + + EXPORT(octeon_mult_restore3_end) + END(octeon_mult_restore3) .set pop diff --git a/arch/mips/kernel/perf_event.c b/arch/mips/kernel/perf_event.c index c1cf9c6c3f77..5d7a9c03903b 100644 --- a/arch/mips/kernel/perf_event.c +++ b/arch/mips/kernel/perf_event.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Linux performance counter support for MIPS. * @@ -8,13 +9,10 @@ * based on the sparc64 perf event code and the x86 code. Performance * counter access is based on the MIPS Oprofile code. And the callchain * support references the code of MIPS stacktrace.c. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include <linux/perf_event.h> +#include <linux/sched/task_stack.h> #include <asm/stacktrace.h> @@ -25,8 +23,8 @@ * the user stack callchains, we will add it here. */ -static void save_raw_perf_callchain(struct perf_callchain_entry *entry, - unsigned long reg29) +static void save_raw_perf_callchain(struct perf_callchain_entry_ctx *entry, + unsigned long reg29) { unsigned long *sp = (unsigned long *)reg29; unsigned long addr; @@ -35,14 +33,14 @@ static void save_raw_perf_callchain(struct perf_callchain_entry *entry, addr = *sp++; if (__kernel_text_address(addr)) { perf_callchain_store(entry, addr); - if (entry->nr >= PERF_MAX_STACK_DEPTH) + if (entry->nr >= entry->max_stack) break; } } } -void perf_callchain_kernel(struct perf_callchain_entry *entry, - struct pt_regs *regs) +void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) { unsigned long sp = regs->regs[29]; #ifdef CONFIG_KALLSYMS @@ -59,7 +57,7 @@ void perf_callchain_kernel(struct perf_callchain_entry *entry, } do { perf_callchain_store(entry, pc); - if (entry->nr >= PERF_MAX_STACK_DEPTH) + if (entry->nr >= entry->max_stack) break; pc = unwind_stack(current, &sp, pc, &ra); } while (pc); diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index 45f1ffcf1a4b..196a070349b0 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Linux performance counter support for MIPS. * @@ -9,10 +10,6 @@ * based on the sparc64 perf event code and the x86 code. Performance * counter access is based on the MIPS Oprofile code. And the callchain * support references the code of MIPS stacktrace.c. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include <linux/cpumask.h> @@ -64,17 +61,11 @@ struct mips_perf_event { #define CNTR_EVEN 0x55555555 #define CNTR_ODD 0xaaaaaaaa #define CNTR_ALL 0xffffffff -#ifdef CONFIG_MIPS_MT_SMP enum { T = 0, V = 1, P = 2, } range; -#else - #define T - #define V - #define P -#endif }; static struct mips_perf_event raw_event; @@ -99,61 +90,46 @@ struct mips_pmu { unsigned int num_counters; }; +static int counter_bits; static struct mips_pmu mipspmu; -#define M_CONFIG1_PC (1 << 4) - -#define M_PERFCTL_EXL (1 << 0) -#define M_PERFCTL_KERNEL (1 << 1) -#define M_PERFCTL_SUPERVISOR (1 << 2) -#define M_PERFCTL_USER (1 << 3) -#define M_PERFCTL_INTERRUPT_ENABLE (1 << 4) -#define M_PERFCTL_EVENT(event) (((event) & 0x3ff) << 5) -#define M_PERFCTL_VPEID(vpe) ((vpe) << 16) +#define M_PERFCTL_EVENT(event) (((event) << MIPS_PERFCTRL_EVENT_S) & \ + MIPS_PERFCTRL_EVENT) +#define M_PERFCTL_VPEID(vpe) ((vpe) << MIPS_PERFCTRL_VPEID_S) #ifdef CONFIG_CPU_BMIPS5000 #define M_PERFCTL_MT_EN(filter) 0 #else /* !CONFIG_CPU_BMIPS5000 */ -#define M_PERFCTL_MT_EN(filter) ((filter) << 20) +#define M_PERFCTL_MT_EN(filter) (filter) #endif /* CONFIG_CPU_BMIPS5000 */ -#define M_TC_EN_ALL M_PERFCTL_MT_EN(0) -#define M_TC_EN_VPE M_PERFCTL_MT_EN(1) -#define M_TC_EN_TC M_PERFCTL_MT_EN(2) -#define M_PERFCTL_TCID(tcid) ((tcid) << 22) -#define M_PERFCTL_WIDE (1 << 30) -#define M_PERFCTL_MORE (1 << 31) -#define M_PERFCTL_TC (1 << 30) +#define M_TC_EN_ALL M_PERFCTL_MT_EN(MIPS_PERFCTRL_MT_EN_ALL) +#define M_TC_EN_VPE M_PERFCTL_MT_EN(MIPS_PERFCTRL_MT_EN_VPE) +#define M_TC_EN_TC M_PERFCTL_MT_EN(MIPS_PERFCTRL_MT_EN_TC) -#define M_PERFCTL_COUNT_EVENT_WHENEVER (M_PERFCTL_EXL | \ - M_PERFCTL_KERNEL | \ - M_PERFCTL_USER | \ - M_PERFCTL_SUPERVISOR | \ - M_PERFCTL_INTERRUPT_ENABLE) +#define M_PERFCTL_COUNT_EVENT_WHENEVER (MIPS_PERFCTRL_EXL | \ + MIPS_PERFCTRL_K | \ + MIPS_PERFCTRL_U | \ + MIPS_PERFCTRL_S | \ + MIPS_PERFCTRL_IE) #ifdef CONFIG_MIPS_MT_SMP #define M_PERFCTL_CONFIG_MASK 0x3fff801f #else #define M_PERFCTL_CONFIG_MASK 0x1f #endif -#define M_PERFCTL_EVENT_MASK 0xfe0 +#define CNTR_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1)) #ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS -static int cpu_has_mipsmt_pertccounters; - static DEFINE_RWLOCK(pmuint_rwlock); #if defined(CONFIG_CPU_BMIPS5000) #define vpe_id() (cpu_has_mipsmt_pertccounters ? \ 0 : (smp_processor_id() & MIPS_CPUID_TO_COUNTER_MASK)) #else -/* - * FIXME: For VSMP, vpe_id() is redefined for Perf-events, because - * cpu_data[cpuid].vpe_id reports 0 for _both_ CPUs. - */ #define vpe_id() (cpu_has_mipsmt_pertccounters ? \ - 0 : smp_processor_id()) + 0 : cpu_vpe_id(¤t_cpu_data)) #endif /* Copied from op_model_mipsxx.c */ @@ -180,6 +156,31 @@ static void pause_local_counters(void); static irqreturn_t mipsxx_pmu_handle_irq(int, void *); static int mipsxx_pmu_handle_shared_irq(void); +/* 0: Not Loongson-3 + * 1: Loongson-3A1000/3B1000/3B1500 + * 2: Loongson-3A2000/3A3000 + * 3: Loongson-3A4000+ + */ + +#define LOONGSON_PMU_TYPE0 0 +#define LOONGSON_PMU_TYPE1 1 +#define LOONGSON_PMU_TYPE2 2 +#define LOONGSON_PMU_TYPE3 3 + +static inline int get_loongson3_pmu_type(void) +{ + if (boot_cpu_type() != CPU_LOONGSON64) + return LOONGSON_PMU_TYPE0; + if ((boot_cpu_data.processor_id & PRID_COMP_MASK) == PRID_COMP_LEGACY) + return LOONGSON_PMU_TYPE1; + if ((boot_cpu_data.processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64C) + return LOONGSON_PMU_TYPE2; + if ((boot_cpu_data.processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64G) + return LOONGSON_PMU_TYPE3; + + return LOONGSON_PMU_TYPE0; +} + static unsigned int mipsxx_pmu_swizzle_perf_idx(unsigned int idx) { if (vpe_id() == 1) @@ -212,17 +213,18 @@ static u64 mipsxx_pmu_read_counter(unsigned int idx) static u64 mipsxx_pmu_read_counter_64(unsigned int idx) { + u64 mask = CNTR_BIT_MASK(counter_bits); idx = mipsxx_pmu_swizzle_perf_idx(idx); switch (idx) { case 0: - return read_c0_perfcntr0_64(); + return read_c0_perfcntr0_64() & mask; case 1: - return read_c0_perfcntr1_64(); + return read_c0_perfcntr1_64() & mask; case 2: - return read_c0_perfcntr2_64(); + return read_c0_perfcntr2_64() & mask; case 3: - return read_c0_perfcntr3_64(); + return read_c0_perfcntr3_64() & mask; default: WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); return 0; @@ -251,6 +253,7 @@ static void mipsxx_pmu_write_counter(unsigned int idx, u64 val) static void mipsxx_pmu_write_counter_64(unsigned int idx, u64 val) { + val &= CNTR_BIT_MASK(counter_bits); idx = mipsxx_pmu_swizzle_perf_idx(idx); switch (idx) { @@ -312,17 +315,21 @@ static int mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc) { int i; + unsigned long cntr_mask; /* * We only need to care the counter mask. The range has been * checked definitely. */ - unsigned long cntr_mask = (hwc->event_base >> 8) & 0xffff; + if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) + cntr_mask = (hwc->event_base >> 10) & 0xffff; + else + cntr_mask = (hwc->event_base >> 8) & 0xffff; for (i = mipspmu.num_counters - 1; i >= 0; i--) { /* * Note that some MIPS perf events can be counted by both - * even and odd counters, wheresas many other are only by + * even and odd counters, whereas many other are only by * even _or_ odd counters. This introduces an issue that * when the former kind of event takes the counter the * latter kind of event wants to use, then the "counter @@ -340,19 +347,46 @@ static int mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc, static void mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct perf_event *event = container_of(evt, struct perf_event, hw); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + unsigned int range = evt->event_base >> 24; WARN_ON(idx < 0 || idx >= mipspmu.num_counters); - cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) | - (evt->config_base & M_PERFCTL_CONFIG_MASK) | - /* Make sure interrupt enabled. */ - M_PERFCTL_INTERRUPT_ENABLE; - if (IS_ENABLED(CONFIG_CPU_BMIPS5000)) + if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) + cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0x3ff) | + (evt->config_base & M_PERFCTL_CONFIG_MASK) | + /* Make sure interrupt enabled. */ + MIPS_PERFCTRL_IE; + else + cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) | + (evt->config_base & M_PERFCTL_CONFIG_MASK) | + /* Make sure interrupt enabled. */ + MIPS_PERFCTRL_IE; + + if (IS_ENABLED(CONFIG_CPU_BMIPS5000)) { /* enable the counter for the calling thread */ cpuc->saved_ctrl[idx] |= - (1 << (12 + vpe_id())) | M_PERFCTL_TC; + (1 << (12 + vpe_id())) | BRCM_PERFCTRL_TC; + } else if (IS_ENABLED(CONFIG_MIPS_MT_SMP) && range > V) { + /* The counter is processor wide. Set it up to count all TCs. */ + pr_debug("Enabling perf counter for all TCs\n"); + cpuc->saved_ctrl[idx] |= M_TC_EN_ALL; + } else { + unsigned int cpu, ctrl; + + /* + * Set up the counter for a particular CPU when event->cpu is + * a valid CPU number. Otherwise set up the counter for the CPU + * scheduling this thread. + */ + cpu = (event->cpu >= 0) ? event->cpu : smp_processor_id(); + ctrl = M_PERFCTL_VPEID(cpu_vpe_id(&cpu_data[cpu])); + ctrl |= M_TC_EN_VPE; + cpuc->saved_ctrl[idx] |= ctrl; + pr_debug("Enabling perf counter for CPU%d\n", cpu); + } /* * We do not actually let the counter run. Leave it until start(). */ @@ -360,7 +394,7 @@ static void mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx) static void mipsxx_pmu_disable_event(int idx) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); unsigned long flags; WARN_ON(idx < 0 || idx >= mipspmu.num_counters); @@ -401,6 +435,10 @@ static int mipspmu_event_set_period(struct perf_event *event, local64_set(&hwc->prev_count, mipspmu.overflow - left); + if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) + mipsxx_pmu_write_control(idx, + M_PERFCTL_EVENT(hwc->event_base & 0x3ff)); + mipspmu.write_counter(idx, mipspmu.overflow - left); perf_event_update_userpage(event); @@ -460,7 +498,7 @@ static void mipspmu_stop(struct perf_event *event, int flags) static int mipspmu_add(struct perf_event *event, int flags) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct hw_perf_event *hwc = &event->hw; int idx; int err = 0; @@ -496,7 +534,7 @@ out: static void mipspmu_del(struct perf_event *event, int flags) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; @@ -530,7 +568,7 @@ static void mipspmu_enable(struct pmu *pmu) /* * MIPS performance counters can be per-TC. The control registers can - * not be directly accessed accross CPUs. Hence if we want to do global + * not be directly accessed across CPUs. Hence if we want to do global * control, we need cross CPU calls. on_each_cpu() can help us, but we * can not make sure this function is called with interrupts enabled. So * here we pause local counters and then grab a rwlock and leave the @@ -558,11 +596,13 @@ static int mipspmu_get_irq(void) if (mipspmu.irq >= 0) { /* Request my own irq handler. */ err = request_irq(mipspmu.irq, mipsxx_pmu_handle_irq, - IRQF_PERCPU | IRQF_NOBALANCING, - "mips_perf_pmu", NULL); + IRQF_PERCPU | IRQF_NOBALANCING | + IRQF_NO_THREAD | IRQF_NO_SUSPEND | + IRQF_SHARED, + "mips_perf_pmu", &mipspmu); if (err) { - pr_warning("Unable to request IRQ%d for MIPS " - "performance counters!\n", mipspmu.irq); + pr_warn("Unable to request IRQ%d for MIPS performance counters!\n", + mipspmu.irq); } } else if (cp0_perfcount_irq < 0) { /* @@ -572,8 +612,7 @@ static int mipspmu_get_irq(void) perf_irq = mipsxx_pmu_handle_shared_irq; err = 0; } else { - pr_warning("The platform hasn't properly defined its " - "interrupt controller.\n"); + pr_warn("The platform hasn't properly defined its interrupt controller\n"); err = -ENOENT; } @@ -583,7 +622,7 @@ static int mipspmu_get_irq(void) static void mipspmu_free_irq(void) { if (mipspmu.irq >= 0) - free_irq(mipspmu.irq, NULL); + free_irq(mipspmu.irq, &mipspmu); else if (cp0_perfcount_irq < 0) perf_irq = save_perf_irq; } @@ -628,8 +667,7 @@ static int mipspmu_event_init(struct perf_event *event) return -ENOENT; } - if (event->cpu >= nr_cpumask_bits || - (event->cpu >= 0 && !cpu_online(event->cpu))) + if (event->cpu >= 0 && !cpu_online(event->cpu)) return -ENODEV; if (!atomic_inc_not_zero(&active_events)) { @@ -666,13 +704,20 @@ static unsigned int mipspmu_perf_event_encode(const struct mips_perf_event *pev) * event_id. */ #ifdef CONFIG_MIPS_MT_SMP - return ((unsigned int)pev->range << 24) | - (pev->cntr_mask & 0xffff00) | - (pev->event_id & 0xff); -#else - return (pev->cntr_mask & 0xffff00) | - (pev->event_id & 0xff); -#endif + if (num_possible_cpus() > 1) + return ((unsigned int)pev->range << 24) | + (pev->cntr_mask & 0xffff00) | + (pev->event_id & 0xff); + else +#endif /* CONFIG_MIPS_MT_SMP */ + { + if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) + return (pev->cntr_mask & 0xfffc00) | + (pev->event_id & 0x3ff); + else + return (pev->cntr_mask & 0xffff00) | + (pev->event_id & 0xff); + } } static const struct mips_perf_event *mipspmu_map_general_event(int idx) @@ -722,7 +767,7 @@ static int validate_group(struct perf_event *event) if (mipsxx_pmu_alloc_counter(&fake_cpuc, &leader->hw) < 0) return -EINVAL; - list_for_each_entry(sibling, &leader->sibling_list, group_entry) { + for_each_sibling_event(sibling, leader) { if (mipsxx_pmu_alloc_counter(&fake_cpuc, &sibling->hw) < 0) return -EINVAL; } @@ -746,20 +791,19 @@ static void handle_associated_event(struct cpu_hw_events *cpuc, if (!mipspmu_event_set_period(event, hwc, idx)) return; - if (perf_event_overflow(event, data, regs)) - mipsxx_pmu_disable_event(idx); + perf_event_overflow(event, data, regs); } static int __n_counters(void) { - if (!(read_c0_config1() & M_CONFIG1_PC)) + if (!cpu_has_perf) return 0; - if (!(read_c0_perfctrl0() & M_PERFCTL_MORE)) + if (!(read_c0_perfctrl0() & MIPS_PERFCTRL_M)) return 1; - if (!(read_c0_perfctrl1() & M_PERFCTL_MORE)) + if (!(read_c0_perfctrl1() & MIPS_PERFCTRL_M)) return 2; - if (!(read_c0_perfctrl2() & M_PERFCTL_MORE)) + if (!(read_c0_perfctrl2() & MIPS_PERFCTRL_M)) return 3; return 4; @@ -776,6 +820,7 @@ static int n_counters(void) case CPU_R12000: case CPU_R14000: + case CPU_R16000: counters = 4; break; @@ -786,26 +831,108 @@ static int n_counters(void) return counters; } +static void loongson3_reset_counters(void *arg) +{ + int counters = (int)(long)arg; + + switch (counters) { + case 4: + mipsxx_pmu_write_control(3, 0); + mipspmu.write_counter(3, 0); + mipsxx_pmu_write_control(3, 127<<5); + mipspmu.write_counter(3, 0); + mipsxx_pmu_write_control(3, 191<<5); + mipspmu.write_counter(3, 0); + mipsxx_pmu_write_control(3, 255<<5); + mipspmu.write_counter(3, 0); + mipsxx_pmu_write_control(3, 319<<5); + mipspmu.write_counter(3, 0); + mipsxx_pmu_write_control(3, 383<<5); + mipspmu.write_counter(3, 0); + mipsxx_pmu_write_control(3, 575<<5); + mipspmu.write_counter(3, 0); + fallthrough; + case 3: + mipsxx_pmu_write_control(2, 0); + mipspmu.write_counter(2, 0); + mipsxx_pmu_write_control(2, 127<<5); + mipspmu.write_counter(2, 0); + mipsxx_pmu_write_control(2, 191<<5); + mipspmu.write_counter(2, 0); + mipsxx_pmu_write_control(2, 255<<5); + mipspmu.write_counter(2, 0); + mipsxx_pmu_write_control(2, 319<<5); + mipspmu.write_counter(2, 0); + mipsxx_pmu_write_control(2, 383<<5); + mipspmu.write_counter(2, 0); + mipsxx_pmu_write_control(2, 575<<5); + mipspmu.write_counter(2, 0); + fallthrough; + case 2: + mipsxx_pmu_write_control(1, 0); + mipspmu.write_counter(1, 0); + mipsxx_pmu_write_control(1, 127<<5); + mipspmu.write_counter(1, 0); + mipsxx_pmu_write_control(1, 191<<5); + mipspmu.write_counter(1, 0); + mipsxx_pmu_write_control(1, 255<<5); + mipspmu.write_counter(1, 0); + mipsxx_pmu_write_control(1, 319<<5); + mipspmu.write_counter(1, 0); + mipsxx_pmu_write_control(1, 383<<5); + mipspmu.write_counter(1, 0); + mipsxx_pmu_write_control(1, 575<<5); + mipspmu.write_counter(1, 0); + fallthrough; + case 1: + mipsxx_pmu_write_control(0, 0); + mipspmu.write_counter(0, 0); + mipsxx_pmu_write_control(0, 127<<5); + mipspmu.write_counter(0, 0); + mipsxx_pmu_write_control(0, 191<<5); + mipspmu.write_counter(0, 0); + mipsxx_pmu_write_control(0, 255<<5); + mipspmu.write_counter(0, 0); + mipsxx_pmu_write_control(0, 319<<5); + mipspmu.write_counter(0, 0); + mipsxx_pmu_write_control(0, 383<<5); + mipspmu.write_counter(0, 0); + mipsxx_pmu_write_control(0, 575<<5); + mipspmu.write_counter(0, 0); + break; + } +} + static void reset_counters(void *arg) { int counters = (int)(long)arg; + + if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) { + loongson3_reset_counters(arg); + return; + } + switch (counters) { case 4: mipsxx_pmu_write_control(3, 0); mipspmu.write_counter(3, 0); + fallthrough; case 3: mipsxx_pmu_write_control(2, 0); mipspmu.write_counter(2, 0); + fallthrough; case 2: mipsxx_pmu_write_control(1, 0); mipspmu.write_counter(1, 0); + fallthrough; case 1: mipsxx_pmu_write_control(0, 0); mipspmu.write_counter(0, 0); + break; } } -/* 24K/34K/1004K cores can share the same event map. */ +/* 24K/34K/1004K/interAptiv/loongson1 cores share the same event map. */ static const struct mips_perf_event mipsxxcore_event_map [PERF_COUNT_HW_MAX] = { [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P }, @@ -814,8 +941,8 @@ static const struct mips_perf_event mipsxxcore_event_map [PERF_COUNT_HW_BRANCH_MISSES] = { 0x02, CNTR_ODD, T }, }; -/* 74K core has different branch event code. */ -static const struct mips_perf_event mipsxx74Kcore_event_map +/* 74K/proAptiv core has different branch event code. */ +static const struct mips_perf_event mipsxxcore_event_map2 [PERF_COUNT_HW_MAX] = { [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P }, [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T }, @@ -823,6 +950,40 @@ static const struct mips_perf_event mipsxx74Kcore_event_map [PERF_COUNT_HW_BRANCH_MISSES] = { 0x27, CNTR_ODD, T }, }; +static const struct mips_perf_event i6x00_event_map[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD }, + /* These only count dcache, not icache */ + [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x45, CNTR_EVEN | CNTR_ODD }, + [PERF_COUNT_HW_CACHE_MISSES] = { 0x48, CNTR_EVEN | CNTR_ODD }, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x15, CNTR_EVEN | CNTR_ODD }, + [PERF_COUNT_HW_BRANCH_MISSES] = { 0x16, CNTR_EVEN | CNTR_ODD }, +}; + +static const struct mips_perf_event loongson3_event_map1[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x00, CNTR_ODD }, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x01, CNTR_EVEN }, + [PERF_COUNT_HW_BRANCH_MISSES] = { 0x01, CNTR_ODD }, +}; + +static const struct mips_perf_event loongson3_event_map2[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = { 0x80, CNTR_ALL }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x81, CNTR_ALL }, + [PERF_COUNT_HW_CACHE_MISSES] = { 0x18, CNTR_ALL }, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x94, CNTR_ALL }, + [PERF_COUNT_HW_BRANCH_MISSES] = { 0x9c, CNTR_ALL }, +}; + +static const struct mips_perf_event loongson3_event_map3[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_ALL }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_ALL }, + [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x1c, CNTR_ALL }, + [PERF_COUNT_HW_CACHE_MISSES] = { 0x1d, CNTR_ALL }, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02, CNTR_ALL }, + [PERF_COUNT_HW_BRANCH_MISSES] = { 0x08, CNTR_ALL }, +}; + static const struct mips_perf_event octeon_event_map[PERF_COUNT_HW_MAX] = { [PERF_COUNT_HW_CPU_CYCLES] = { 0x01, CNTR_ALL }, [PERF_COUNT_HW_INSTRUCTIONS] = { 0x03, CNTR_ALL }, @@ -840,16 +1001,7 @@ static const struct mips_perf_event bmips5000_event_map [PERF_COUNT_HW_BRANCH_MISSES] = { 0x02, CNTR_ODD, T }, }; -static const struct mips_perf_event xlp_event_map[PERF_COUNT_HW_MAX] = { - [PERF_COUNT_HW_CPU_CYCLES] = { 0x01, CNTR_ALL }, - [PERF_COUNT_HW_INSTRUCTIONS] = { 0x18, CNTR_ALL }, /* PAPI_TOT_INS */ - [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x04, CNTR_ALL }, /* PAPI_L1_ICA */ - [PERF_COUNT_HW_CACHE_MISSES] = { 0x07, CNTR_ALL }, /* PAPI_L1_ICM */ - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x1b, CNTR_ALL }, /* PAPI_BR_CN */ - [PERF_COUNT_HW_BRANCH_MISSES] = { 0x1c, CNTR_ALL }, /* PAPI_BR_MSP */ -}; - -/* 24K/34K/1004K cores can share the same cache event map. */ +/* 24K/34K/1004K/interAptiv/loongson1 cores share the same cache event map. */ static const struct mips_perf_event mipsxxcore_cache_map [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] @@ -930,8 +1082,8 @@ static const struct mips_perf_event mipsxxcore_cache_map }, }; -/* 74K core has completely different cache event map. */ -static const struct mips_perf_event mipsxx74Kcore_cache_map +/* 74K/proAptiv core has completely different cache event map. */ +static const struct mips_perf_event mipsxxcore_cache_map2 [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX] = { @@ -971,13 +1123,18 @@ static const struct mips_perf_event mipsxx74Kcore_cache_map [C(LL)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = { 0x1c, CNTR_ODD, P }, - [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN | CNTR_ODD, P }, + [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN, P }, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = { 0x1c, CNTR_ODD, P }, - [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN | CNTR_ODD, P }, + [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN, P }, }, }, +/* + * 74K core does not have specific DTLB events. proAptiv core has + * "speculative" DTLB events which are numbered 0x63 (even/odd) and + * not included here. One can use raw events if really needed. + */ [C(ITLB)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = { 0x04, CNTR_EVEN, T }, @@ -1001,8 +1158,47 @@ static const struct mips_perf_event mipsxx74Kcore_cache_map }, }; -/* BMIPS5000 */ -static const struct mips_perf_event bmips5000_cache_map +static const struct mips_perf_event i6x00_cache_map + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { +[C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x46, CNTR_EVEN | CNTR_ODD }, + [C(RESULT_MISS)] = { 0x49, CNTR_EVEN | CNTR_ODD }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x47, CNTR_EVEN | CNTR_ODD }, + [C(RESULT_MISS)] = { 0x4a, CNTR_EVEN | CNTR_ODD }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x84, CNTR_EVEN | CNTR_ODD }, + [C(RESULT_MISS)] = { 0x85, CNTR_EVEN | CNTR_ODD }, + }, +}, +[C(DTLB)] = { + /* Can't distinguish read & write */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x40, CNTR_EVEN | CNTR_ODD }, + [C(RESULT_MISS)] = { 0x41, CNTR_EVEN | CNTR_ODD }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x40, CNTR_EVEN | CNTR_ODD }, + [C(RESULT_MISS)] = { 0x41, CNTR_EVEN | CNTR_ODD }, + }, +}, +[C(BPU)] = { + /* Conditional branches / mispredicted */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x15, CNTR_EVEN | CNTR_ODD }, + [C(RESULT_MISS)] = { 0x16, CNTR_EVEN | CNTR_ODD }, + }, +}, +}; + +static const struct mips_perf_event loongson3_cache_map1 [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX] = { @@ -1014,180 +1210,262 @@ static const struct mips_perf_event bmips5000_cache_map * best we can do. Writes and reads get combined. */ [C(OP_READ)] = { - [C(RESULT_ACCESS)] = { 12, CNTR_EVEN, T }, - [C(RESULT_MISS)] = { 12, CNTR_ODD, T }, + [C(RESULT_MISS)] = { 0x04, CNTR_ODD }, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = { 12, CNTR_EVEN, T }, - [C(RESULT_MISS)] = { 12, CNTR_ODD, T }, + [C(RESULT_MISS)] = { 0x04, CNTR_ODD }, }, }, [C(L1I)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = { 10, CNTR_EVEN, T }, - [C(RESULT_MISS)] = { 10, CNTR_ODD, T }, + [C(RESULT_MISS)] = { 0x04, CNTR_EVEN }, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = { 10, CNTR_EVEN, T }, - [C(RESULT_MISS)] = { 10, CNTR_ODD, T }, + [C(RESULT_MISS)] = { 0x04, CNTR_EVEN }, }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = { 23, CNTR_EVEN, T }, - /* - * Note that MIPS has only "hit" events countable for - * the prefetch operation. - */ +}, +[C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x09, CNTR_ODD }, + }, + [C(OP_WRITE)] = { + [C(RESULT_MISS)] = { 0x09, CNTR_ODD }, }, }, -[C(LL)] = { +[C(ITLB)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = { 28, CNTR_EVEN, P }, - [C(RESULT_MISS)] = { 28, CNTR_ODD, P }, + [C(RESULT_MISS)] = { 0x0c, CNTR_ODD }, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = { 28, CNTR_EVEN, P }, - [C(RESULT_MISS)] = { 28, CNTR_ODD, P }, + [C(RESULT_MISS)] = { 0x0c, CNTR_ODD }, }, }, [C(BPU)] = { /* Using the same code for *HW_BRANCH* */ [C(OP_READ)] = { - [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T }, + [C(RESULT_ACCESS)] = { 0x01, CNTR_EVEN }, + [C(RESULT_MISS)] = { 0x01, CNTR_ODD }, }, [C(OP_WRITE)] = { - [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T }, + [C(RESULT_ACCESS)] = { 0x01, CNTR_EVEN }, + [C(RESULT_MISS)] = { 0x01, CNTR_ODD }, }, }, }; - -static const struct mips_perf_event octeon_cache_map +static const struct mips_perf_event loongson3_cache_map2 [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX] = { [C(L1D)] = { + /* + * Like some other architectures (e.g. ARM), the performance + * counters don't differentiate between read and write + * accesses/misses, so this isn't strictly correct, but it's the + * best we can do. Writes and reads get combined. + */ [C(OP_READ)] = { - [C(RESULT_ACCESS)] = { 0x2b, CNTR_ALL }, - [C(RESULT_MISS)] = { 0x2e, CNTR_ALL }, + [C(RESULT_ACCESS)] = { 0x156, CNTR_ALL }, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = { 0x30, CNTR_ALL }, + [C(RESULT_ACCESS)] = { 0x155, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x153, CNTR_ALL }, }, }, [C(L1I)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = { 0x18, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x18, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_MISS)] = { 0x18, CNTR_ALL }, + }, +}, +[C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x1b6, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x1b7, CNTR_ALL }, }, [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = { 0x19, CNTR_ALL }, + [C(RESULT_ACCESS)] = { 0x1bf, CNTR_ALL }, }, }, [C(DTLB)] = { - /* - * Only general DTLB misses are counted use the same event for - * read and write. - */ [C(OP_READ)] = { - [C(RESULT_MISS)] = { 0x35, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x92, CNTR_ALL }, }, [C(OP_WRITE)] = { - [C(RESULT_MISS)] = { 0x35, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x92, CNTR_ALL }, }, }, [C(ITLB)] = { [C(OP_READ)] = { - [C(RESULT_MISS)] = { 0x37, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x1a, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_MISS)] = { 0x1a, CNTR_ALL }, + }, +}, +[C(BPU)] = { + /* Using the same code for *HW_BRANCH* */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x94, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x9c, CNTR_ALL }, }, }, }; -static const struct mips_perf_event xlp_cache_map +static const struct mips_perf_event loongson3_cache_map3 [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX] = { [C(L1D)] = { + /* + * Like some other architectures (e.g. ARM), the performance + * counters don't differentiate between read and write + * accesses/misses, so this isn't strictly correct, but it's the + * best we can do. Writes and reads get combined. + */ [C(OP_READ)] = { - [C(RESULT_ACCESS)] = { 0x31, CNTR_ALL }, /* PAPI_L1_DCR */ - [C(RESULT_MISS)] = { 0x30, CNTR_ALL }, /* PAPI_L1_LDM */ + [C(RESULT_ACCESS)] = { 0x1e, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x1f, CNTR_ALL }, }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = { 0x2f, CNTR_ALL }, /* PAPI_L1_DCW */ - [C(RESULT_MISS)] = { 0x2e, CNTR_ALL }, /* PAPI_L1_STM */ + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { 0xaa, CNTR_ALL }, + [C(RESULT_MISS)] = { 0xa9, CNTR_ALL }, }, }, [C(L1I)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = { 0x04, CNTR_ALL }, /* PAPI_L1_ICA */ - [C(RESULT_MISS)] = { 0x07, CNTR_ALL }, /* PAPI_L1_ICM */ + [C(RESULT_ACCESS)] = { 0x1c, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x1d, CNTR_ALL }, }, }, [C(LL)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = { 0x35, CNTR_ALL }, /* PAPI_L2_DCR */ - [C(RESULT_MISS)] = { 0x37, CNTR_ALL }, /* PAPI_L2_LDM */ - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = { 0x34, CNTR_ALL }, /* PAPI_L2_DCA */ - [C(RESULT_MISS)] = { 0x36, CNTR_ALL }, /* PAPI_L2_DCM */ + [C(RESULT_ACCESS)] = { 0x2e, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x2f, CNTR_ALL }, }, }, [C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x14, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x1b, CNTR_ALL }, + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x1a, CNTR_ALL }, + }, +}, +[C(BPU)] = { + /* Using the same code for *HW_BRANCH* */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x02, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x08, CNTR_ALL }, + }, +}, +}; + +/* BMIPS5000 */ +static const struct mips_perf_event bmips5000_cache_map + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { +[C(L1D)] = { /* - * Only general DTLB misses are counted use the same event for - * read and write. + * Like some other architectures (e.g. ARM), the performance + * counters don't differentiate between read and write + * accesses/misses, so this isn't strictly correct, but it's the + * best we can do. Writes and reads get combined. */ [C(OP_READ)] = { - [C(RESULT_MISS)] = { 0x2d, CNTR_ALL }, /* PAPI_TLB_DM */ + [C(RESULT_ACCESS)] = { 12, CNTR_EVEN, T }, + [C(RESULT_MISS)] = { 12, CNTR_ODD, T }, }, [C(OP_WRITE)] = { - [C(RESULT_MISS)] = { 0x2d, CNTR_ALL }, /* PAPI_TLB_DM */ + [C(RESULT_ACCESS)] = { 12, CNTR_EVEN, T }, + [C(RESULT_MISS)] = { 12, CNTR_ODD, T }, }, }, -[C(ITLB)] = { +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 10, CNTR_EVEN, T }, + [C(RESULT_MISS)] = { 10, CNTR_ODD, T }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 10, CNTR_EVEN, T }, + [C(RESULT_MISS)] = { 10, CNTR_ODD, T }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { 23, CNTR_EVEN, T }, + /* + * Note that MIPS has only "hit" events countable for + * the prefetch operation. + */ + }, +}, +[C(LL)] = { [C(OP_READ)] = { - [C(RESULT_MISS)] = { 0x08, CNTR_ALL }, /* PAPI_TLB_IM */ + [C(RESULT_ACCESS)] = { 28, CNTR_EVEN, P }, + [C(RESULT_MISS)] = { 28, CNTR_ODD, P }, }, [C(OP_WRITE)] = { - [C(RESULT_MISS)] = { 0x08, CNTR_ALL }, /* PAPI_TLB_IM */ + [C(RESULT_ACCESS)] = { 28, CNTR_EVEN, P }, + [C(RESULT_MISS)] = { 28, CNTR_ODD, P }, }, }, [C(BPU)] = { + /* Using the same code for *HW_BRANCH* */ [C(OP_READ)] = { - [C(RESULT_MISS)] = { 0x25, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T }, + }, + [C(OP_WRITE)] = { + [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T }, }, }, }; -#ifdef CONFIG_MIPS_MT_SMP -static void check_and_calc_range(struct perf_event *event, - const struct mips_perf_event *pev) -{ - struct hw_perf_event *hwc = &event->hw; - - if (event->cpu >= 0) { - if (pev->range > V) { - /* - * The user selected an event that is processor - * wide, while expecting it to be VPE wide. - */ - hwc->config_base |= M_TC_EN_ALL; - } else { - /* - * FIXME: cpu_data[event->cpu].vpe_id reports 0 - * for both CPUs. - */ - hwc->config_base |= M_PERFCTL_VPEID(event->cpu); - hwc->config_base |= M_TC_EN_VPE; - } - } else - hwc->config_base |= M_TC_EN_ALL; -} -#else -static void check_and_calc_range(struct perf_event *event, - const struct mips_perf_event *pev) -{ -} -#endif +static const struct mips_perf_event octeon_cache_map + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { +[C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x2b, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x2e, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x30, CNTR_ALL }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x18, CNTR_ALL }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { 0x19, CNTR_ALL }, + }, +}, +[C(DTLB)] = { + /* + * Only general DTLB misses are counted use the same event for + * read and write. + */ + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x35, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_MISS)] = { 0x35, CNTR_ALL }, + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x37, CNTR_ALL }, + }, +}, +}; static int __hw_perf_event_init(struct perf_event *event) { @@ -1222,25 +1500,21 @@ static int __hw_perf_event_init(struct perf_event *event) * We allow max flexibility on how each individual counter shared * by the single CPU operates (the mode exclusion and the range). */ - hwc->config_base = M_PERFCTL_INTERRUPT_ENABLE; - - /* Calculate range bits and validate it. */ - if (num_possible_cpus() > 1) - check_and_calc_range(event, pev); + hwc->config_base = MIPS_PERFCTRL_IE; hwc->event_base = mipspmu_perf_event_encode(pev); if (PERF_TYPE_RAW == event->attr.type) mutex_unlock(&raw_event_mutex); if (!attr->exclude_user) - hwc->config_base |= M_PERFCTL_USER; + hwc->config_base |= MIPS_PERFCTRL_U; if (!attr->exclude_kernel) { - hwc->config_base |= M_PERFCTL_KERNEL; + hwc->config_base |= MIPS_PERFCTRL_K; /* MIPS kernel mode: KSU == 00b || EXL == 1 || ERL == 1 */ - hwc->config_base |= M_PERFCTL_EXL; + hwc->config_base |= MIPS_PERFCTRL_EXL; } if (!attr->exclude_hv) - hwc->config_base |= M_PERFCTL_SUPERVISOR; + hwc->config_base |= MIPS_PERFCTRL_S; hwc->config_base &= M_PERFCTL_CONFIG_MASK; /* @@ -1270,7 +1544,7 @@ static int __hw_perf_event_init(struct perf_event *event) static void pause_local_counters(void) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int ctr = mipspmu.num_counters; unsigned long flags; @@ -1286,7 +1560,7 @@ static void pause_local_counters(void) static void resume_local_counters(void) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); int ctr = mipspmu.num_counters; do { @@ -1297,11 +1571,11 @@ static void resume_local_counters(void) static int mipsxx_pmu_handle_shared_irq(void) { - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct perf_sample_data data; unsigned int counters = mipspmu.num_counters; u64 counter; - int handled = IRQ_NONE; + int n, handled = IRQ_NONE; struct pt_regs *regs; if (cpu_has_perf_cntr_intr_bit && !(read_c0_cause() & CAUSEF_PCI)) @@ -1322,22 +1596,23 @@ static int mipsxx_pmu_handle_shared_irq(void) perf_sample_data_init(&data, 0, 0); - switch (counters) { -#define HANDLE_COUNTER(n) \ - case n + 1: \ - if (test_bit(n, cpuc->used_mask)) { \ - counter = mipspmu.read_counter(n); \ - if (counter & mipspmu.overflow) { \ - handle_associated_event(cpuc, n, &data, regs); \ - handled = IRQ_HANDLED; \ - } \ - } - HANDLE_COUNTER(3) - HANDLE_COUNTER(2) - HANDLE_COUNTER(1) - HANDLE_COUNTER(0) + for (n = counters - 1; n >= 0; n--) { + if (!test_bit(n, cpuc->used_mask)) + continue; + + counter = mipspmu.read_counter(n); + if (!(counter & mipspmu.overflow)) + continue; + + handle_associated_event(cpuc, n, &data, regs); + handled = IRQ_HANDLED; } +#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS + read_unlock(&pmuint_rwlock); +#endif + resume_local_counters(); + /* * Do all the work for the pending perf events. We can do this * in here because the performance counter interrupt is a regular @@ -1346,10 +1621,6 @@ static int mipsxx_pmu_handle_shared_irq(void) if (handled == IRQ_HANDLED) irq_work_run(); -#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS - read_unlock(&pmuint_rwlock); -#endif - resume_local_counters(); return handled; } @@ -1378,6 +1649,13 @@ static irqreturn_t mipsxx_pmu_handle_irq(int irq, void *dev) #define IS_BOTH_COUNTERS_74K_EVENT(b) \ ((b) == 0 || (b) == 1) +/* proAptiv */ +#define IS_BOTH_COUNTERS_PROAPTIV_EVENT(b) \ + ((b) == 0 || (b) == 1) +/* P5600 */ +#define IS_BOTH_COUNTERS_P5600_EVENT(b) \ + ((b) == 0 || (b) == 1) + /* 1004K */ #define IS_BOTH_COUNTERS_1004K_EVENT(b) \ ((b) == 0 || (b) == 1 || (b) == 11) @@ -1391,26 +1669,44 @@ static irqreturn_t mipsxx_pmu_handle_irq(int irq, void *dev) #define IS_RANGE_V_1004K_EVENT(r) ((r) == 47) #endif +/* interAptiv */ +#define IS_BOTH_COUNTERS_INTERAPTIV_EVENT(b) \ + ((b) == 0 || (b) == 1 || (b) == 11) +#ifdef CONFIG_MIPS_MT_SMP +/* The P/V/T info is not provided for "(b) == 38" in SUM, assume P. */ +#define IS_RANGE_P_INTERAPTIV_EVENT(r, b) \ + ((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 || \ + (b) == 25 || (b) == 36 || (b) == 38 || (b) == 39 || \ + (r) == 44 || (r) == 174 || (r) == 176 || ((b) >= 50 && \ + (b) <= 59) || (r) == 188 || (b) == 61 || (b) == 62 || \ + ((b) >= 64 && (b) <= 67)) +#define IS_RANGE_V_INTERAPTIV_EVENT(r) ((r) == 47 || (r) == 175) +#endif + /* BMIPS5000 */ #define IS_BOTH_COUNTERS_BMIPS5000_EVENT(b) \ ((b) == 0 || (b) == 1) /* - * User can use 0-255 raw events, where 0-127 for the events of even - * counters, and 128-255 for odd counters. Note that bit 7 is used to - * indicate the parity. So, for example, when user wants to take the - * Event Num of 15 for odd counters (by referring to the user manual), - * then 128 needs to be added to 15 as the input for the event config, - * i.e., 143 (0x8F) to be used. + * For most cores the user can use 0-255 raw events, where 0-127 for the events + * of even counters, and 128-255 for odd counters. Note that bit 7 is used to + * indicate the even/odd bank selector. So, for example, when user wants to take + * the Event Num of 15 for odd counters (by referring to the user manual), then + * 128 needs to be added to 15 as the input for the event config, i.e., 143 (0x8F) + * to be used. + * + * Some newer cores have even more events, in which case the user can use raw + * events 0-511, where 0-255 are for the events of even counters, and 256-511 + * are for odd counters, so bit 8 is used to indicate the even/odd bank selector. */ static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config) { + /* currently most cores have 7-bit event numbers */ + int pmu_type; unsigned int raw_id = config & 0xff; unsigned int base_id = raw_id & 0x7f; - raw_event.event_id = base_id; - switch (current_cpu_type()) { case CPU_24K: if (IS_BOTH_COUNTERS_24K_EVENT(base_id)) @@ -1442,6 +1738,7 @@ static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config) #endif break; case CPU_74K: + case CPU_1074K: if (IS_BOTH_COUNTERS_74K_EVENT(base_id)) raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; else @@ -1451,6 +1748,36 @@ static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config) raw_event.range = P; #endif break; + case CPU_PROAPTIV: + if (IS_BOTH_COUNTERS_PROAPTIV_EVENT(base_id)) + raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; + else + raw_event.cntr_mask = + raw_id > 127 ? CNTR_ODD : CNTR_EVEN; +#ifdef CONFIG_MIPS_MT_SMP + raw_event.range = P; +#endif + break; + case CPU_P5600: + case CPU_P6600: + /* 8-bit event numbers */ + raw_id = config & 0x1ff; + base_id = raw_id & 0xff; + if (IS_BOTH_COUNTERS_P5600_EVENT(base_id)) + raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; + else + raw_event.cntr_mask = + raw_id > 255 ? CNTR_ODD : CNTR_EVEN; +#ifdef CONFIG_MIPS_MT_SMP + raw_event.range = P; +#endif + break; + case CPU_I6400: + case CPU_I6500: + /* 8-bit event numbers */ + base_id = config & 0xff; + raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; + break; case CPU_1004K: if (IS_BOTH_COUNTERS_1004K_EVENT(base_id)) raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; @@ -1466,32 +1793,81 @@ static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config) raw_event.range = T; #endif break; + case CPU_INTERAPTIV: + if (IS_BOTH_COUNTERS_INTERAPTIV_EVENT(base_id)) + raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; + else + raw_event.cntr_mask = + raw_id > 127 ? CNTR_ODD : CNTR_EVEN; +#ifdef CONFIG_MIPS_MT_SMP + if (IS_RANGE_P_INTERAPTIV_EVENT(raw_id, base_id)) + raw_event.range = P; + else if (unlikely(IS_RANGE_V_INTERAPTIV_EVENT(raw_id))) + raw_event.range = V; + else + raw_event.range = T; +#endif + break; case CPU_BMIPS5000: if (IS_BOTH_COUNTERS_BMIPS5000_EVENT(base_id)) raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; else raw_event.cntr_mask = raw_id > 127 ? CNTR_ODD : CNTR_EVEN; + break; + case CPU_LOONGSON64: + pmu_type = get_loongson3_pmu_type(); + + switch (pmu_type) { + case LOONGSON_PMU_TYPE1: + raw_event.cntr_mask = + raw_id > 127 ? CNTR_ODD : CNTR_EVEN; + break; + case LOONGSON_PMU_TYPE2: + base_id = config & 0x3ff; + raw_event.cntr_mask = CNTR_ALL; + + if ((base_id >= 1 && base_id < 28) || + (base_id >= 64 && base_id < 90) || + (base_id >= 128 && base_id < 164) || + (base_id >= 192 && base_id < 200) || + (base_id >= 256 && base_id < 275) || + (base_id >= 320 && base_id < 361) || + (base_id >= 384 && base_id < 574)) + break; + + return ERR_PTR(-EOPNOTSUPP); + case LOONGSON_PMU_TYPE3: + base_id = raw_id; + raw_event.cntr_mask = CNTR_ALL; + break; + } + break; } + raw_event.event_id = base_id; + return &raw_event; } static const struct mips_perf_event *octeon_pmu_map_raw_event(u64 config) { - unsigned int raw_id = config & 0xff; - unsigned int base_id = raw_id & 0x7f; + unsigned int base_id = config & 0x7f; + unsigned int event_max; raw_event.cntr_mask = CNTR_ALL; raw_event.event_id = base_id; - if (current_cpu_type() == CPU_CAVIUM_OCTEON2) { - if (base_id > 0x42) - return ERR_PTR(-EOPNOTSUPP); - } else { - if (base_id > 0x3a) - return ERR_PTR(-EOPNOTSUPP); + if (current_cpu_type() == CPU_CAVIUM_OCTEON3) + event_max = 0x5f; + else if (current_cpu_type() == CPU_CAVIUM_OCTEON2) + event_max = 0x42; + else + event_max = 0x3a; + + if (base_id > event_max) { + return ERR_PTR(-EOPNOTSUPP); } switch (base_id) { @@ -1501,7 +1877,7 @@ static const struct mips_perf_event *octeon_pmu_map_raw_event(u64 config) case 0x1f: case 0x2f: case 0x34: - case 0x3b ... 0x3f: + case 0x3e ... 0x3f: return ERR_PTR(-EOPNOTSUPP); default: break; @@ -1510,25 +1886,10 @@ static const struct mips_perf_event *octeon_pmu_map_raw_event(u64 config) return &raw_event; } -static const struct mips_perf_event *xlp_pmu_map_raw_event(u64 config) -{ - unsigned int raw_id = config & 0xff; - - /* Only 1-63 are defined */ - if ((raw_id < 0x01) || (raw_id > 0x3f)) - return ERR_PTR(-EOPNOTSUPP); - - raw_event.cntr_mask = CNTR_ALL; - raw_event.event_id = raw_id; - - return &raw_event; -} - static int __init init_hw_perf_events(void) { - int counters, irq; - int counter_bits; + int counters, irq, pmu_type; pr_info("Performance counters: "); @@ -1539,27 +1900,16 @@ init_hw_perf_events(void) } #ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS - cpu_has_mipsmt_pertccounters = read_c0_config7() & (1<<19); if (!cpu_has_mipsmt_pertccounters) counters = counters_total_to_per_cpu(counters); #endif -#ifdef MSC01E_INT_BASE - if (cpu_has_veic) { - /* - * Using platform specific interrupt controller defines. - */ - irq = MSC01E_INT_BASE + MSC01E_INT_PERFCTR; - } else { -#endif - if ((cp0_perfcount_irq >= 0) && - (cp0_compare_irq != cp0_perfcount_irq)) - irq = MIPS_CPU_IRQ_BASE + cp0_perfcount_irq; - else - irq = -1; -#ifdef MSC01E_INT_BASE - } -#endif + if (get_c0_perfcount_int) + irq = get_c0_perfcount_int(); + else if (cp0_perfcount_irq >= 0) + irq = MIPS_CPU_IRQ_BASE + cp0_perfcount_irq; + else + irq = -1; mipspmu.map_raw_event = mipsxx_pmu_map_raw_event; @@ -1576,22 +1926,80 @@ init_hw_perf_events(void) break; case CPU_74K: mipspmu.name = "mips/74K"; - mipspmu.general_event_map = &mipsxx74Kcore_event_map; - mipspmu.cache_event_map = &mipsxx74Kcore_cache_map; + mipspmu.general_event_map = &mipsxxcore_event_map2; + mipspmu.cache_event_map = &mipsxxcore_cache_map2; + break; + case CPU_PROAPTIV: + mipspmu.name = "mips/proAptiv"; + mipspmu.general_event_map = &mipsxxcore_event_map2; + mipspmu.cache_event_map = &mipsxxcore_cache_map2; + break; + case CPU_P5600: + mipspmu.name = "mips/P5600"; + mipspmu.general_event_map = &mipsxxcore_event_map2; + mipspmu.cache_event_map = &mipsxxcore_cache_map2; + break; + case CPU_P6600: + mipspmu.name = "mips/P6600"; + mipspmu.general_event_map = &mipsxxcore_event_map2; + mipspmu.cache_event_map = &mipsxxcore_cache_map2; + break; + case CPU_I6400: + mipspmu.name = "mips/I6400"; + mipspmu.general_event_map = &i6x00_event_map; + mipspmu.cache_event_map = &i6x00_cache_map; + break; + case CPU_I6500: + mipspmu.name = "mips/I6500"; + mipspmu.general_event_map = &i6x00_event_map; + mipspmu.cache_event_map = &i6x00_cache_map; break; case CPU_1004K: mipspmu.name = "mips/1004K"; mipspmu.general_event_map = &mipsxxcore_event_map; mipspmu.cache_event_map = &mipsxxcore_cache_map; break; - case CPU_LOONGSON1: + case CPU_1074K: + mipspmu.name = "mips/1074K"; + mipspmu.general_event_map = &mipsxxcore_event_map; + mipspmu.cache_event_map = &mipsxxcore_cache_map; + break; + case CPU_INTERAPTIV: + mipspmu.name = "mips/interAptiv"; + mipspmu.general_event_map = &mipsxxcore_event_map; + mipspmu.cache_event_map = &mipsxxcore_cache_map; + break; + case CPU_LOONGSON32: mipspmu.name = "mips/loongson1"; mipspmu.general_event_map = &mipsxxcore_event_map; mipspmu.cache_event_map = &mipsxxcore_cache_map; break; + case CPU_LOONGSON64: + mipspmu.name = "mips/loongson3"; + pmu_type = get_loongson3_pmu_type(); + + switch (pmu_type) { + case LOONGSON_PMU_TYPE1: + counters = 2; + mipspmu.general_event_map = &loongson3_event_map1; + mipspmu.cache_event_map = &loongson3_cache_map1; + break; + case LOONGSON_PMU_TYPE2: + counters = 4; + mipspmu.general_event_map = &loongson3_event_map2; + mipspmu.cache_event_map = &loongson3_cache_map2; + break; + case LOONGSON_PMU_TYPE3: + counters = 4; + mipspmu.general_event_map = &loongson3_event_map3; + mipspmu.cache_event_map = &loongson3_cache_map3; + break; + } + break; case CPU_CAVIUM_OCTEON: case CPU_CAVIUM_OCTEON_PLUS: case CPU_CAVIUM_OCTEON2: + case CPU_CAVIUM_OCTEON3: mipspmu.name = "octeon"; mipspmu.general_event_map = &octeon_event_map; mipspmu.cache_event_map = &octeon_cache_map; @@ -1602,12 +2010,6 @@ init_hw_perf_events(void) mipspmu.general_event_map = &bmips5000_event_map; mipspmu.cache_event_map = &bmips5000_cache_map; break; - case CPU_XLP: - mipspmu.name = "xlp"; - mipspmu.general_event_map = &xlp_event_map; - mipspmu.cache_event_map = &xlp_cache_map; - mipspmu.map_raw_event = xlp_pmu_map_raw_event; - break; default: pr_cont("Either hardware does not support performance " "counters, or not yet implemented.\n"); @@ -1617,20 +2019,27 @@ init_hw_perf_events(void) mipspmu.num_counters = counters; mipspmu.irq = irq; - if (read_c0_perfctrl0() & M_PERFCTL_WIDE) { - mipspmu.max_period = (1ULL << 63) - 1; - mipspmu.valid_count = (1ULL << 63) - 1; - mipspmu.overflow = 1ULL << 63; + if (read_c0_perfctrl0() & MIPS_PERFCTRL_W) { + if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) { + counter_bits = 48; + mipspmu.max_period = (1ULL << 47) - 1; + mipspmu.valid_count = (1ULL << 47) - 1; + mipspmu.overflow = 1ULL << 47; + } else { + counter_bits = 64; + mipspmu.max_period = (1ULL << 63) - 1; + mipspmu.valid_count = (1ULL << 63) - 1; + mipspmu.overflow = 1ULL << 63; + } mipspmu.read_counter = mipsxx_pmu_read_counter_64; mipspmu.write_counter = mipsxx_pmu_write_counter_64; - counter_bits = 64; } else { + counter_bits = 32; mipspmu.max_period = (1ULL << 31) - 1; mipspmu.valid_count = (1ULL << 31) - 1; mipspmu.overflow = 1ULL << 31; mipspmu.read_counter = mipsxx_pmu_read_counter; mipspmu.write_counter = mipsxx_pmu_write_counter; - counter_bits = 32; } on_each_cpu(reset_counters, (void *)(long)counters, 1); diff --git a/arch/mips/kernel/perf_regs.c b/arch/mips/kernel/perf_regs.c new file mode 100644 index 000000000000..e686780d1647 --- /dev/null +++ b/arch/mips/kernel/perf_regs.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Some parts derived from x86 version of this file. + * + * Copyright (C) 2013 Cavium, Inc. + */ + +#include <linux/perf_event.h> + +#include <asm/ptrace.h> + +#ifdef CONFIG_32BIT +u64 perf_reg_abi(struct task_struct *tsk) +{ + return PERF_SAMPLE_REGS_ABI_32; +} +#else /* Must be CONFIG_64BIT */ +u64 perf_reg_abi(struct task_struct *tsk) +{ + if (test_tsk_thread_flag(tsk, TIF_32BIT_REGS)) + return PERF_SAMPLE_REGS_ABI_32; + else + return PERF_SAMPLE_REGS_ABI_64; +} +#endif /* CONFIG_32BIT */ + +int perf_reg_validate(u64 mask) +{ + if (!mask) + return -EINVAL; + if (mask & ~((1ull << PERF_REG_MIPS_MAX) - 1)) + return -EINVAL; + return 0; +} + +u64 perf_reg_value(struct pt_regs *regs, int idx) +{ + long v; + + switch (idx) { + case PERF_REG_MIPS_PC: + v = regs->cp0_epc; + break; + case PERF_REG_MIPS_R1 ... PERF_REG_MIPS_R25: + v = regs->regs[idx - PERF_REG_MIPS_R1 + 1]; + break; + case PERF_REG_MIPS_R28 ... PERF_REG_MIPS_R31: + v = regs->regs[idx - PERF_REG_MIPS_R28 + 28]; + break; + + default: + WARN_ON_ONCE(1); + return 0; + } + + return (s64)v; /* Sign extend if 32-bit. */ +} + +void perf_get_regs_user(struct perf_regs *regs_user, + struct pt_regs *regs) +{ + regs_user->regs = task_pt_regs(current); + regs_user->abi = perf_reg_abi(current); +} diff --git a/arch/mips/kernel/pm-cps.c b/arch/mips/kernel/pm-cps.c new file mode 100644 index 000000000000..3de0e05e0511 --- /dev/null +++ b/arch/mips/kernel/pm-cps.c @@ -0,0 +1,739 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2014 Imagination Technologies + * Author: Paul Burton <paul.burton@mips.com> + */ + +#include <linux/cpuhotplug.h> +#include <linux/init.h> +#include <linux/percpu.h> +#include <linux/slab.h> +#include <linux/suspend.h> + +#include <asm/asm-offsets.h> +#include <asm/cacheflush.h> +#include <asm/cacheops.h> +#include <asm/idle.h> +#include <asm/mips-cps.h> +#include <asm/mipsmtregs.h> +#include <asm/pm.h> +#include <asm/pm-cps.h> +#include <asm/regdef.h> +#include <asm/smp-cps.h> +#include <asm/uasm.h> + +/* + * cps_nc_entry_fn - type of a generated non-coherent state entry function + * @online: the count of online coupled VPEs + * @nc_ready_count: pointer to a non-coherent mapping of the core ready_count + * + * The code entering & exiting non-coherent states is generated at runtime + * using uasm, in order to ensure that the compiler cannot insert a stray + * memory access at an unfortunate time and to allow the generation of optimal + * core-specific code particularly for cache routines. If coupled_coherence + * is non-zero and this is the entry function for the CPS_PM_NC_WAIT state, + * returns the number of VPEs that were in the wait state at the point this + * VPE left it. Returns garbage if coupled_coherence is zero or this is not + * the entry function for CPS_PM_NC_WAIT. + */ +typedef unsigned (*cps_nc_entry_fn)(unsigned online, u32 *nc_ready_count); + +/* + * The entry point of the generated non-coherent idle state entry/exit + * functions. Actually per-core rather than per-CPU. + */ +static DEFINE_PER_CPU_READ_MOSTLY(cps_nc_entry_fn[CPS_PM_STATE_COUNT], + nc_asm_enter); + +/* Bitmap indicating which states are supported by the system */ +static DECLARE_BITMAP(state_support, CPS_PM_STATE_COUNT); + +/* + * Indicates the number of coupled VPEs ready to operate in a non-coherent + * state. Actually per-core rather than per-CPU. + */ +static DEFINE_PER_CPU_ALIGNED(u32*, ready_count); + +/* Indicates online CPUs coupled with the current CPU */ +static DEFINE_PER_CPU_ALIGNED(cpumask_t, online_coupled); + +/* Used to synchronize entry to deep idle states */ +static DEFINE_PER_CPU_ALIGNED(atomic_t, pm_barrier); + +/* Saved CPU state across the CPS_PM_POWER_GATED state */ +DEFINE_PER_CPU_ALIGNED(struct mips_static_suspend_state, cps_cpu_state); + +/* A somewhat arbitrary number of labels & relocs for uasm */ +static struct uasm_label labels[32]; +static struct uasm_reloc relocs[32]; + +bool cps_pm_support_state(enum cps_pm_state state) +{ + return test_bit(state, state_support); +} + +static void coupled_barrier(atomic_t *a, unsigned online) +{ + /* + * This function is effectively the same as + * cpuidle_coupled_parallel_barrier, which can't be used here since + * there's no cpuidle device. + */ + + if (!coupled_coherence) + return; + + smp_mb__before_atomic(); + atomic_inc(a); + + while (atomic_read(a) < online) + cpu_relax(); + + if (atomic_inc_return(a) == online * 2) { + atomic_set(a, 0); + return; + } + + while (atomic_read(a) > online) + cpu_relax(); +} + +int cps_pm_enter_state(enum cps_pm_state state) +{ + unsigned cpu = smp_processor_id(); + unsigned int cluster = cpu_cluster(¤t_cpu_data); + unsigned core = cpu_core(¤t_cpu_data); + unsigned online, left; + cpumask_t *coupled_mask = this_cpu_ptr(&online_coupled); + u32 *core_ready_count, *nc_core_ready_count; + void *nc_addr; + cps_nc_entry_fn entry; + struct cluster_boot_config *cluster_cfg; + struct core_boot_config *core_cfg; + struct vpe_boot_config *vpe_cfg; + atomic_t *barrier; + + /* Check that there is an entry function for this state */ + entry = per_cpu(nc_asm_enter, cpu)[state]; + if (!entry) + return -EINVAL; + + /* Calculate which coupled CPUs (VPEs) are online */ +#if defined(CONFIG_MIPS_MT) || defined(CONFIG_CPU_MIPSR6) + if (cpu_online(cpu)) { + cpumask_and(coupled_mask, cpu_online_mask, + &cpu_sibling_map[cpu]); + online = cpumask_weight(coupled_mask); + cpumask_clear_cpu(cpu, coupled_mask); + } else +#endif + { + cpumask_clear(coupled_mask); + online = 1; + } + + /* Setup the VPE to run mips_cps_pm_restore when started again */ + if (IS_ENABLED(CONFIG_CPU_PM) && state == CPS_PM_POWER_GATED) { + /* Power gating relies upon CPS SMP */ + if (!mips_cps_smp_in_use()) + return -EINVAL; + + cluster_cfg = &mips_cps_cluster_bootcfg[cluster]; + core_cfg = &cluster_cfg->core_config[core]; + vpe_cfg = &core_cfg->vpe_config[cpu_vpe_id(¤t_cpu_data)]; + vpe_cfg->pc = (unsigned long)mips_cps_pm_restore; + vpe_cfg->gp = (unsigned long)current_thread_info(); + vpe_cfg->sp = 0; + } + + /* Indicate that this CPU might not be coherent */ + cpumask_clear_cpu(cpu, &cpu_coherent_mask); + smp_mb__after_atomic(); + + /* Create a non-coherent mapping of the core ready_count */ + core_ready_count = per_cpu(ready_count, cpu); + nc_addr = kmap_noncoherent(virt_to_page(core_ready_count), + (unsigned long)core_ready_count); + nc_addr += ((unsigned long)core_ready_count & ~PAGE_MASK); + nc_core_ready_count = nc_addr; + + /* Ensure ready_count is zero-initialised before the assembly runs */ + WRITE_ONCE(*nc_core_ready_count, 0); + barrier = &per_cpu(pm_barrier, cpumask_first(&cpu_sibling_map[cpu])); + coupled_barrier(barrier, online); + + /* Run the generated entry code */ + left = entry(online, nc_core_ready_count); + + /* Remove the non-coherent mapping of ready_count */ + kunmap_noncoherent(); + + /* Indicate that this CPU is definitely coherent */ + cpumask_set_cpu(cpu, &cpu_coherent_mask); + + /* + * If this VPE is the first to leave the non-coherent wait state then + * it needs to wake up any coupled VPEs still running their wait + * instruction so that they return to cpuidle, which can then complete + * coordination between the coupled VPEs & provide the governor with + * a chance to reflect on the length of time the VPEs were in the + * idle state. + */ + if (coupled_coherence && (state == CPS_PM_NC_WAIT) && (left == online)) + arch_send_call_function_ipi_mask(coupled_mask); + + return 0; +} + +static void cps_gen_cache_routine(u32 **pp, struct uasm_label **pl, + struct uasm_reloc **pr, + const struct cache_desc *cache, + unsigned op, int lbl) +{ + unsigned cache_size = cache->ways << cache->waybit; + unsigned i; + const unsigned unroll_lines = 32; + + /* If the cache isn't present this function has it easy */ + if (cache->flags & MIPS_CACHE_NOT_PRESENT) + return; + + /* Load base address */ + UASM_i_LA(pp, GPR_T0, (long)CKSEG0); + + /* Calculate end address */ + if (cache_size < 0x8000) + uasm_i_addiu(pp, GPR_T1, GPR_T0, cache_size); + else + UASM_i_LA(pp, GPR_T1, (long)(CKSEG0 + cache_size)); + + /* Start of cache op loop */ + uasm_build_label(pl, *pp, lbl); + + /* Generate the cache ops */ + for (i = 0; i < unroll_lines; i++) { + if (cpu_has_mips_r6) { + uasm_i_cache(pp, op, 0, GPR_T0); + uasm_i_addiu(pp, GPR_T0, GPR_T0, cache->linesz); + } else { + uasm_i_cache(pp, op, i * cache->linesz, GPR_T0); + } + } + + if (!cpu_has_mips_r6) + /* Update the base address */ + uasm_i_addiu(pp, GPR_T0, GPR_T0, unroll_lines * cache->linesz); + + /* Loop if we haven't reached the end address yet */ + uasm_il_bne(pp, pr, GPR_T0, GPR_T1, lbl); + uasm_i_nop(pp); +} + +static int cps_gen_flush_fsb(u32 **pp, struct uasm_label **pl, + struct uasm_reloc **pr, + const struct cpuinfo_mips *cpu_info, + int lbl) +{ + unsigned i, fsb_size = 8; + unsigned num_loads = (fsb_size * 3) / 2; + unsigned line_stride = 2; + unsigned line_size = cpu_info->dcache.linesz; + unsigned perf_counter, perf_event; + unsigned revision = cpu_info->processor_id & PRID_REV_MASK; + + /* + * Determine whether this CPU requires an FSB flush, and if so which + * performance counter/event reflect stalls due to a full FSB. + */ + switch (__get_cpu_type(cpu_info->cputype)) { + case CPU_INTERAPTIV: + perf_counter = 1; + perf_event = 51; + break; + + case CPU_PROAPTIV: + /* Newer proAptiv cores don't require this workaround */ + if (revision >= PRID_REV_ENCODE_332(1, 1, 0)) + return 0; + + /* On older ones it's unavailable */ + return -1; + + default: + /* Assume that the CPU does not need this workaround */ + return 0; + } + + /* + * Ensure that the fill/store buffer (FSB) is not holding the results + * of a prefetch, since if it is then the CPC sequencer may become + * stuck in the D3 (ClrBus) state whilst entering a low power state. + */ + + /* Preserve perf counter setup */ + uasm_i_mfc0(pp, GPR_T2, 25, (perf_counter * 2) + 0); /* PerfCtlN */ + uasm_i_mfc0(pp, GPR_T3, 25, (perf_counter * 2) + 1); /* PerfCntN */ + + /* Setup perf counter to count FSB full pipeline stalls */ + uasm_i_addiu(pp, GPR_T0, GPR_ZERO, (perf_event << 5) | 0xf); + uasm_i_mtc0(pp, GPR_T0, 25, (perf_counter * 2) + 0); /* PerfCtlN */ + uasm_i_ehb(pp); + uasm_i_mtc0(pp, GPR_ZERO, 25, (perf_counter * 2) + 1); /* PerfCntN */ + uasm_i_ehb(pp); + + /* Base address for loads */ + UASM_i_LA(pp, GPR_T0, (long)CKSEG0); + + /* Start of clear loop */ + uasm_build_label(pl, *pp, lbl); + + /* Perform some loads to fill the FSB */ + for (i = 0; i < num_loads; i++) + uasm_i_lw(pp, GPR_ZERO, i * line_size * line_stride, GPR_T0); + + /* + * Invalidate the new D-cache entries so that the cache will need + * refilling (via the FSB) if the loop is executed again. + */ + for (i = 0; i < num_loads; i++) { + uasm_i_cache(pp, Hit_Invalidate_D, + i * line_size * line_stride, GPR_T0); + uasm_i_cache(pp, Hit_Writeback_Inv_SD, + i * line_size * line_stride, GPR_T0); + } + + /* Barrier ensuring previous cache invalidates are complete */ + uasm_i_sync(pp, __SYNC_full); + uasm_i_ehb(pp); + + /* Check whether the pipeline stalled due to the FSB being full */ + uasm_i_mfc0(pp, GPR_T1, 25, (perf_counter * 2) + 1); /* PerfCntN */ + + /* Loop if it didn't */ + uasm_il_beqz(pp, pr, GPR_T1, lbl); + uasm_i_nop(pp); + + /* Restore perf counter 1. The count may well now be wrong... */ + uasm_i_mtc0(pp, GPR_T2, 25, (perf_counter * 2) + 0); /* PerfCtlN */ + uasm_i_ehb(pp); + uasm_i_mtc0(pp, GPR_T3, 25, (perf_counter * 2) + 1); /* PerfCntN */ + uasm_i_ehb(pp); + + return 0; +} + +static void cps_gen_set_top_bit(u32 **pp, struct uasm_label **pl, + struct uasm_reloc **pr, + unsigned r_addr, int lbl) +{ + uasm_i_lui(pp, GPR_T0, uasm_rel_hi(0x80000000)); + uasm_build_label(pl, *pp, lbl); + uasm_i_ll(pp, GPR_T1, 0, r_addr); + uasm_i_or(pp, GPR_T1, GPR_T1, GPR_T0); + uasm_i_sc(pp, GPR_T1, 0, r_addr); + uasm_il_beqz(pp, pr, GPR_T1, lbl); + uasm_i_nop(pp); +} + +static void *cps_gen_entry_code(unsigned cpu, enum cps_pm_state state) +{ + struct uasm_label *l = labels; + struct uasm_reloc *r = relocs; + u32 *buf, *p; + const unsigned r_online = GPR_A0; + const unsigned r_nc_count = GPR_A1; + const unsigned r_pcohctl = GPR_T8; + const unsigned max_instrs = 256; + unsigned cpc_cmd; + int err; + enum { + lbl_incready = 1, + lbl_poll_cont, + lbl_secondary_hang, + lbl_disable_coherence, + lbl_flush_fsb, + lbl_invicache, + lbl_flushdcache, + lbl_hang, + lbl_set_cont, + lbl_secondary_cont, + lbl_decready, + }; + + /* Allocate a buffer to hold the generated code */ + p = buf = kcalloc(max_instrs, sizeof(u32), GFP_KERNEL); + if (!buf) + return NULL; + + /* Clear labels & relocs ready for (re)use */ + memset(labels, 0, sizeof(labels)); + memset(relocs, 0, sizeof(relocs)); + + if (IS_ENABLED(CONFIG_CPU_PM) && state == CPS_PM_POWER_GATED) { + /* Power gating relies upon CPS SMP */ + if (!mips_cps_smp_in_use()) + goto out_err; + + /* + * Save CPU state. Note the non-standard calling convention + * with the return address placed in v0 to avoid clobbering + * the ra register before it is saved. + */ + UASM_i_LA(&p, GPR_T0, (long)mips_cps_pm_save); + uasm_i_jalr(&p, GPR_V0, GPR_T0); + uasm_i_nop(&p); + } + + /* + * Load addresses of required CM & CPC registers. This is done early + * because they're needed in both the enable & disable coherence steps + * but in the coupled case the enable step will only run on one VPE. + */ + UASM_i_LA(&p, r_pcohctl, (long)addr_gcr_cl_coherence()); + + if (coupled_coherence) { + /* Increment ready_count */ + uasm_i_sync(&p, __SYNC_mb); + uasm_build_label(&l, p, lbl_incready); + uasm_i_ll(&p, GPR_T1, 0, r_nc_count); + uasm_i_addiu(&p, GPR_T2, GPR_T1, 1); + uasm_i_sc(&p, GPR_T2, 0, r_nc_count); + uasm_il_beqz(&p, &r, GPR_T2, lbl_incready); + uasm_i_addiu(&p, GPR_T1, GPR_T1, 1); + + /* Barrier ensuring all CPUs see the updated r_nc_count value */ + uasm_i_sync(&p, __SYNC_mb); + + /* + * If this is the last VPE to become ready for non-coherence + * then it should branch below. + */ + uasm_il_beq(&p, &r, GPR_T1, r_online, lbl_disable_coherence); + uasm_i_nop(&p); + + if (state < CPS_PM_POWER_GATED) { + /* + * Otherwise this is not the last VPE to become ready + * for non-coherence. It needs to wait until coherence + * has been disabled before proceeding, which it will do + * by polling for the top bit of ready_count being set. + */ + uasm_i_addiu(&p, GPR_T1, GPR_ZERO, -1); + uasm_build_label(&l, p, lbl_poll_cont); + uasm_i_lw(&p, GPR_T0, 0, r_nc_count); + uasm_il_bltz(&p, &r, GPR_T0, lbl_secondary_cont); + uasm_i_ehb(&p); + if (cpu_has_mipsmt) + uasm_i_yield(&p, GPR_ZERO, GPR_T1); + uasm_il_b(&p, &r, lbl_poll_cont); + uasm_i_nop(&p); + } else { + /* + * The core will lose power & this VPE will not continue + * so it can simply halt here. + */ + if (cpu_has_mipsmt) { + /* Halt the VPE via C0 tchalt register */ + uasm_i_addiu(&p, GPR_T0, GPR_ZERO, TCHALT_H); + uasm_i_mtc0(&p, GPR_T0, 2, 4); + } else if (cpu_has_vp) { + /* Halt the VP via the CPC VP_STOP register */ + unsigned int vpe_id; + + vpe_id = cpu_vpe_id(&cpu_data[cpu]); + uasm_i_addiu(&p, GPR_T0, GPR_ZERO, 1 << vpe_id); + UASM_i_LA(&p, GPR_T1, (long)addr_cpc_cl_vp_stop()); + uasm_i_sw(&p, GPR_T0, 0, GPR_T1); + } else { + BUG(); + } + uasm_build_label(&l, p, lbl_secondary_hang); + uasm_il_b(&p, &r, lbl_secondary_hang); + uasm_i_nop(&p); + } + } + + /* + * This is the point of no return - this VPE will now proceed to + * disable coherence. At this point we *must* be sure that no other + * VPE within the core will interfere with the L1 dcache. + */ + uasm_build_label(&l, p, lbl_disable_coherence); + + /* Invalidate the L1 icache */ + cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].icache, + Index_Invalidate_I, lbl_invicache); + + /* Writeback & invalidate the L1 dcache */ + cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].dcache, + Index_Writeback_Inv_D, lbl_flushdcache); + + /* Barrier ensuring previous cache invalidates are complete */ + uasm_i_sync(&p, __SYNC_full); + uasm_i_ehb(&p); + + if (mips_cm_revision() < CM_REV_CM3) { + /* + * Disable all but self interventions. The load from COHCTL is + * defined by the interAptiv & proAptiv SUMs as ensuring that the + * operation resulting from the preceding store is complete. + */ + uasm_i_addiu(&p, GPR_T0, GPR_ZERO, 1 << cpu_core(&cpu_data[cpu])); + uasm_i_sw(&p, GPR_T0, 0, r_pcohctl); + uasm_i_lw(&p, GPR_T0, 0, r_pcohctl); + + /* Barrier to ensure write to coherence control is complete */ + uasm_i_sync(&p, __SYNC_full); + uasm_i_ehb(&p); + } + + /* Disable coherence */ + uasm_i_sw(&p, GPR_ZERO, 0, r_pcohctl); + uasm_i_lw(&p, GPR_T0, 0, r_pcohctl); + + if (state >= CPS_PM_CLOCK_GATED) { + err = cps_gen_flush_fsb(&p, &l, &r, &cpu_data[cpu], + lbl_flush_fsb); + if (err) + goto out_err; + + /* Determine the CPC command to issue */ + switch (state) { + case CPS_PM_CLOCK_GATED: + cpc_cmd = CPC_Cx_CMD_CLOCKOFF; + break; + case CPS_PM_POWER_GATED: + cpc_cmd = CPC_Cx_CMD_PWRDOWN; + break; + default: + BUG(); + goto out_err; + } + + /* Issue the CPC command */ + UASM_i_LA(&p, GPR_T0, (long)addr_cpc_cl_cmd()); + uasm_i_addiu(&p, GPR_T1, GPR_ZERO, cpc_cmd); + uasm_i_sw(&p, GPR_T1, 0, GPR_T0); + + if (state == CPS_PM_POWER_GATED) { + /* If anything goes wrong just hang */ + uasm_build_label(&l, p, lbl_hang); + uasm_il_b(&p, &r, lbl_hang); + uasm_i_nop(&p); + + /* + * There's no point generating more code, the core is + * powered down & if powered back up will run from the + * reset vector not from here. + */ + goto gen_done; + } + + /* Barrier to ensure write to CPC command is complete */ + uasm_i_sync(&p, __SYNC_full); + uasm_i_ehb(&p); + } + + if (state == CPS_PM_NC_WAIT) { + /* + * At this point it is safe for all VPEs to proceed with + * execution. This VPE will set the top bit of ready_count + * to indicate to the other VPEs that they may continue. + */ + if (coupled_coherence) + cps_gen_set_top_bit(&p, &l, &r, r_nc_count, + lbl_set_cont); + + /* + * VPEs which did not disable coherence will continue + * executing, after coherence has been disabled, from this + * point. + */ + uasm_build_label(&l, p, lbl_secondary_cont); + + /* Now perform our wait */ + uasm_i_wait(&p, 0); + } + + /* + * Re-enable coherence. Note that for CPS_PM_NC_WAIT all coupled VPEs + * will run this. The first will actually re-enable coherence & the + * rest will just be performing a rather unusual nop. + */ + uasm_i_addiu(&p, GPR_T0, GPR_ZERO, mips_cm_revision() < CM_REV_CM3 + ? CM_GCR_Cx_COHERENCE_COHDOMAINEN + : CM3_GCR_Cx_COHERENCE_COHEN); + + uasm_i_sw(&p, GPR_T0, 0, r_pcohctl); + uasm_i_lw(&p, GPR_T0, 0, r_pcohctl); + + /* Barrier to ensure write to coherence control is complete */ + uasm_i_sync(&p, __SYNC_full); + uasm_i_ehb(&p); + + if (coupled_coherence && (state == CPS_PM_NC_WAIT)) { + /* Decrement ready_count */ + uasm_build_label(&l, p, lbl_decready); + uasm_i_sync(&p, __SYNC_mb); + uasm_i_ll(&p, GPR_T1, 0, r_nc_count); + uasm_i_addiu(&p, GPR_T2, GPR_T1, -1); + uasm_i_sc(&p, GPR_T2, 0, r_nc_count); + uasm_il_beqz(&p, &r, GPR_T2, lbl_decready); + uasm_i_andi(&p, GPR_V0, GPR_T1, (1 << fls(smp_num_siblings)) - 1); + + /* Barrier ensuring all CPUs see the updated r_nc_count value */ + uasm_i_sync(&p, __SYNC_mb); + } + + if (coupled_coherence && (state == CPS_PM_CLOCK_GATED)) { + /* + * At this point it is safe for all VPEs to proceed with + * execution. This VPE will set the top bit of ready_count + * to indicate to the other VPEs that they may continue. + */ + cps_gen_set_top_bit(&p, &l, &r, r_nc_count, lbl_set_cont); + + /* + * This core will be reliant upon another core sending a + * power-up command to the CPC in order to resume operation. + * Thus an arbitrary VPE can't trigger the core leaving the + * idle state and the one that disables coherence might as well + * be the one to re-enable it. The rest will continue from here + * after that has been done. + */ + uasm_build_label(&l, p, lbl_secondary_cont); + + /* Barrier ensuring all CPUs see the updated r_nc_count value */ + uasm_i_sync(&p, __SYNC_mb); + } + + /* The core is coherent, time to return to C code */ + uasm_i_jr(&p, GPR_RA); + uasm_i_nop(&p); + +gen_done: + /* Ensure the code didn't exceed the resources allocated for it */ + BUG_ON((p - buf) > max_instrs); + BUG_ON((l - labels) > ARRAY_SIZE(labels)); + BUG_ON((r - relocs) > ARRAY_SIZE(relocs)); + + /* Patch branch offsets */ + uasm_resolve_relocs(relocs, labels); + + /* Flush the icache */ + local_flush_icache_range((unsigned long)buf, (unsigned long)p); + + return buf; +out_err: + kfree(buf); + return NULL; +} + +static int cps_pm_online_cpu(unsigned int cpu) +{ + unsigned int sibling, core; + void *entry_fn, *core_rc; + enum cps_pm_state state; + + core = cpu_core(&cpu_data[cpu]); + + for (state = CPS_PM_NC_WAIT; state < CPS_PM_STATE_COUNT; state++) { + if (per_cpu(nc_asm_enter, cpu)[state]) + continue; + if (!test_bit(state, state_support)) + continue; + + entry_fn = cps_gen_entry_code(cpu, state); + if (!entry_fn) { + pr_err("Failed to generate core %u state %u entry\n", + core, state); + clear_bit(state, state_support); + } + + for_each_cpu(sibling, &cpu_sibling_map[cpu]) + per_cpu(nc_asm_enter, sibling)[state] = entry_fn; + } + + if (!per_cpu(ready_count, cpu)) { + core_rc = kmalloc(sizeof(u32), GFP_KERNEL); + if (!core_rc) { + pr_err("Failed allocate core %u ready_count\n", core); + return -ENOMEM; + } + + for_each_cpu(sibling, &cpu_sibling_map[cpu]) + per_cpu(ready_count, sibling) = core_rc; + } + + return 0; +} + +static int cps_pm_power_notifier(struct notifier_block *this, + unsigned long event, void *ptr) +{ + unsigned int stat; + + switch (event) { + case PM_SUSPEND_PREPARE: + stat = read_cpc_cl_stat_conf(); + /* + * If we're attempting to suspend the system and power down all + * of the cores, the JTAG detect bit indicates that the CPC will + * instead put the cores into clock-off state. In this state + * a connected debugger can cause the CPU to attempt + * interactions with the powered down system. At best this will + * fail. At worst, it can hang the NoC, requiring a hard reset. + * To avoid this, just block system suspend if a JTAG probe + * is detected. + */ + if (stat & CPC_Cx_STAT_CONF_EJTAG_PROBE) { + pr_warn("JTAG probe is connected - abort suspend\n"); + return NOTIFY_BAD; + } + return NOTIFY_DONE; + default: + return NOTIFY_DONE; + } +} + +static int __init cps_pm_init(void) +{ + /* A CM is required for all non-coherent states */ + if (!mips_cm_present()) { + pr_warn("pm-cps: no CM, non-coherent states unavailable\n"); + return 0; + } + + /* + * If interrupts were enabled whilst running a wait instruction on a + * non-coherent core then the VPE may end up processing interrupts + * whilst non-coherent. That would be bad. + */ + if (cpu_wait == r4k_wait_irqoff) + set_bit(CPS_PM_NC_WAIT, state_support); + else + pr_warn("pm-cps: non-coherent wait unavailable\n"); + + /* Detect whether a CPC is present */ + if (mips_cpc_present()) { + /* Detect whether clock gating is implemented */ + if (read_cpc_cl_stat_conf() & CPC_Cx_STAT_CONF_CLKGAT_IMPL) + set_bit(CPS_PM_CLOCK_GATED, state_support); + else + pr_warn("pm-cps: CPC does not support clock gating\n"); + + /* Power gating is available with CPS SMP & any CPC */ + if (mips_cps_smp_in_use()) + set_bit(CPS_PM_POWER_GATED, state_support); + else + pr_warn("pm-cps: CPS SMP not in use, power gating unavailable\n"); + } else { + pr_warn("pm-cps: no CPC, clock & power gating unavailable\n"); + } + + pm_notifier(cps_pm_power_notifier, 0); + + return cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mips/cps_pm:online", + cps_pm_online_cpu, NULL); +} +arch_initcall(cps_pm_init); diff --git a/arch/mips/kernel/pm.c b/arch/mips/kernel/pm.c new file mode 100644 index 000000000000..486ed2bf2514 --- /dev/null +++ b/arch/mips/kernel/pm.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2014 Imagination Technologies Ltd. + * + * CPU PM notifiers for saving/restoring general CPU state. + */ + +#include <linux/cpu_pm.h> +#include <linux/init.h> + +#include <asm/dsp.h> +#include <asm/fpu.h> +#include <asm/mmu_context.h> +#include <asm/pm.h> +#include <asm/watch.h> + +/* Used by PM helper macros in asm/pm.h */ +struct mips_static_suspend_state mips_static_suspend_state; + +/** + * mips_cpu_save() - Save general CPU state. + * Ensures that general CPU context is saved, notably FPU and DSP. + */ +static int mips_cpu_save(void) +{ + /* Save FPU state */ + lose_fpu(1); + + /* Save DSP state */ + save_dsp(current); + + return 0; +} + +/** + * mips_cpu_restore() - Restore general CPU state. + * Restores important CPU context. + */ +static void mips_cpu_restore(void) +{ + unsigned int cpu = smp_processor_id(); + + /* Restore ASID */ + if (current->mm) + write_c0_entryhi(cpu_asid(cpu, current->mm)); + + /* Restore DSP state */ + restore_dsp(current); + + /* Restore UserLocal */ + if (cpu_has_userlocal) + write_c0_userlocal(current_thread_info()->tp_value); + + /* Restore watch registers */ + __restore_watch(current); +} + +/** + * mips_pm_notifier() - Notifier for preserving general CPU context. + * @self: Notifier block. + * @cmd: CPU PM event. + * @v: Private data (unused). + * + * This is called when a CPU power management event occurs, and is used to + * ensure that important CPU context is preserved across a CPU power down. + */ +static int mips_pm_notifier(struct notifier_block *self, unsigned long cmd, + void *v) +{ + int ret; + + switch (cmd) { + case CPU_PM_ENTER: + ret = mips_cpu_save(); + if (ret) + return NOTIFY_STOP; + break; + case CPU_PM_ENTER_FAILED: + case CPU_PM_EXIT: + mips_cpu_restore(); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block mips_pm_notifier_block = { + .notifier_call = mips_pm_notifier, +}; + +static int __init mips_pm_init(void) +{ + return cpu_pm_register_notifier(&mips_pm_notifier_block); +} +arch_initcall(mips_pm_init); diff --git a/arch/mips/kernel/probes-common.h b/arch/mips/kernel/probes-common.h new file mode 100644 index 000000000000..73e1d5e95e19 --- /dev/null +++ b/arch/mips/kernel/probes-common.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2016 Imagination Technologies + * Author: Marcin Nowakowski <marcin.nowakowski@mips.com> + */ + +#ifndef __PROBES_COMMON_H +#define __PROBES_COMMON_H + +#include <asm/inst.h> + +int __insn_is_compact_branch(union mips_instruction insn); + +static inline int __insn_has_delay_slot(const union mips_instruction insn) +{ + switch (insn.i_format.opcode) { + /* + * jr and jalr are in r_format format. + */ + case spec_op: + switch (insn.r_format.func) { + case jalr_op: + case jr_op: + return 1; + } + break; + + /* + * This group contains: + * bltz_op, bgez_op, bltzl_op, bgezl_op, + * bltzal_op, bgezal_op, bltzall_op, bgezall_op. + */ + case bcond_op: + switch (insn.i_format.rt) { + case bltz_op: + case bltzl_op: + case bgez_op: + case bgezl_op: + case bltzal_op: + case bltzall_op: + case bgezal_op: + case bgezall_op: + case bposge32_op: + return 1; + } + break; + + /* + * These are unconditional and in j_format. + */ + case jal_op: + case j_op: + case beq_op: + case beql_op: + case bne_op: + case bnel_op: + case blez_op: /* not really i_format */ + case blezl_op: + case bgtz_op: + case bgtzl_op: + return 1; + + /* + * And now the FPA/cp1 branch instructions. + */ + case cop1_op: +#ifdef CONFIG_CPU_CAVIUM_OCTEON + case lwc2_op: /* This is bbit0 on Octeon */ + case ldc2_op: /* This is bbit032 on Octeon */ + case swc2_op: /* This is bbit1 on Octeon */ + case sdc2_op: /* This is bbit132 on Octeon */ +#endif + return 1; + } + + return 0; +} + +#endif /* __PROBES_COMMON_H */ diff --git a/arch/mips/kernel/proc.c b/arch/mips/kernel/proc.c index 8c58d8a84bf3..8f0a0001540c 100644 --- a/arch/mips/kernel/proc.c +++ b/arch/mips/kernel/proc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 1995, 1996, 2001 Ralf Baechle * Copyright (C) 2001, 2004 MIPS Technologies, Inc. @@ -17,12 +18,28 @@ unsigned int vced_count, vcei_count; +/* + * No lock; only written during early bootup by CPU 0. + */ +static RAW_NOTIFIER_HEAD(proc_cpuinfo_chain); + +int __ref register_proc_cpuinfo_notifier(struct notifier_block *nb) +{ + return raw_notifier_chain_register(&proc_cpuinfo_chain, nb); +} + +int proc_cpuinfo_notifier_call_chain(unsigned long val, void *v) +{ + return raw_notifier_call_chain(&proc_cpuinfo_chain, val, v); +} + static int show_cpuinfo(struct seq_file *m, void *v) { + struct proc_cpuinfo_notifier_args proc_cpuinfo_notifier_args; unsigned long n = (unsigned long) v - 1; unsigned int version = cpu_data[n].processor_id; unsigned int fp_vers = cpu_data[n].fpu_id; - char fmt [64]; + char fmt[64]; int i; #ifdef CONFIG_SMP @@ -49,70 +66,242 @@ static int show_cpuinfo(struct seq_file *m, void *v) seq_printf(m, "BogoMIPS\t\t: %u.%02u\n", cpu_data[n].udelay_val / (500000/HZ), (cpu_data[n].udelay_val / (5000/HZ)) % 100); - seq_printf(m, "wait instruction\t: %s\n", cpu_wait ? "yes" : "no"); + seq_printf(m, "wait instruction\t: %s\n", str_yes_no(cpu_wait)); seq_printf(m, "microsecond timers\t: %s\n", - cpu_has_counter ? "yes" : "no"); + str_yes_no(cpu_has_counter)); seq_printf(m, "tlb_entries\t\t: %d\n", cpu_data[n].tlbsize); seq_printf(m, "extra interrupt vector\t: %s\n", - cpu_has_divec ? "yes" : "no"); - seq_printf(m, "hardware watchpoint\t: %s", - cpu_has_watch ? "yes, " : "no\n"); + str_yes_no(cpu_has_divec)); + seq_printf(m, "hardware watchpoint\t: %s", str_yes_no(cpu_has_watch)); if (cpu_has_watch) { - seq_printf(m, "count: %d, address/irw mask: [", + seq_printf(m, ", count: %d, address/irw mask: [", cpu_data[n].watch_reg_count); for (i = 0; i < cpu_data[n].watch_reg_count; i++) - seq_printf(m, "%s0x%04x", i ? ", " : "" , + seq_printf(m, "%s0x%04x", i ? ", " : "", cpu_data[n].watch_reg_masks[i]); - seq_printf(m, "]\n"); - } - if (cpu_has_mips_r) { - seq_printf(m, "isa\t\t\t: mips1"); - if (cpu_has_mips_2) - seq_printf(m, "%s", " mips2"); - if (cpu_has_mips_3) - seq_printf(m, "%s", " mips3"); - if (cpu_has_mips_4) - seq_printf(m, "%s", " mips4"); - if (cpu_has_mips_5) - seq_printf(m, "%s", " mips5"); - if (cpu_has_mips32r1) - seq_printf(m, "%s", " mips32r1"); - if (cpu_has_mips32r2) - seq_printf(m, "%s", " mips32r2"); - if (cpu_has_mips64r1) - seq_printf(m, "%s", " mips64r1"); - if (cpu_has_mips64r2) - seq_printf(m, "%s", " mips64r2"); - seq_printf(m, "\n"); + seq_puts(m, "]"); } - seq_printf(m, "ASEs implemented\t:"); - if (cpu_has_mips16) seq_printf(m, "%s", " mips16"); - if (cpu_has_mdmx) seq_printf(m, "%s", " mdmx"); - if (cpu_has_mips3d) seq_printf(m, "%s", " mips3d"); - if (cpu_has_smartmips) seq_printf(m, "%s", " smartmips"); - if (cpu_has_dsp) seq_printf(m, "%s", " dsp"); - if (cpu_has_dsp2) seq_printf(m, "%s", " dsp2"); - if (cpu_has_mipsmt) seq_printf(m, "%s", " mt"); - if (cpu_has_mmips) seq_printf(m, "%s", " micromips"); - if (cpu_has_vz) seq_printf(m, "%s", " vz"); - seq_printf(m, "\n"); + seq_puts(m, "\nisa\t\t\t:"); + if (cpu_has_mips_1) + seq_puts(m, " mips1"); + if (cpu_has_mips_2) + seq_puts(m, " mips2"); + if (cpu_has_mips_3) + seq_puts(m, " mips3"); + if (cpu_has_mips_4) + seq_puts(m, " mips4"); + if (cpu_has_mips_5) + seq_puts(m, " mips5"); + if (cpu_has_mips32r1) + seq_puts(m, " mips32r1"); + if (cpu_has_mips32r2) + seq_puts(m, " mips32r2"); + if (cpu_has_mips32r5) + seq_puts(m, " mips32r5"); + if (cpu_has_mips32r6) + seq_puts(m, " mips32r6"); + if (cpu_has_mips64r1) + seq_puts(m, " mips64r1"); + if (cpu_has_mips64r2) + seq_puts(m, " mips64r2"); + if (cpu_has_mips64r5) + seq_puts(m, " mips64r5"); + if (cpu_has_mips64r6) + seq_puts(m, " mips64r6"); + seq_puts(m, "\n"); + + seq_puts(m, "ASEs implemented\t:"); + if (cpu_has_mips16) + seq_puts(m, " mips16"); + if (cpu_has_mips16e2) + seq_puts(m, " mips16e2"); + if (cpu_has_mdmx) + seq_puts(m, " mdmx"); + if (cpu_has_mips3d) + seq_puts(m, " mips3d"); + if (cpu_has_smartmips) + seq_puts(m, " smartmips"); + if (cpu_has_dsp) + seq_puts(m, " dsp"); + if (cpu_has_dsp2) + seq_puts(m, " dsp2"); + if (cpu_has_dsp3) + seq_puts(m, " dsp3"); + if (cpu_has_mipsmt) + seq_puts(m, " mt"); + if (cpu_has_mmips) + seq_puts(m, " micromips"); + if (cpu_has_vz) + seq_puts(m, " vz"); + if (cpu_has_msa) + seq_puts(m, " msa"); + if (cpu_has_eva) + seq_puts(m, " eva"); + if (cpu_has_htw) + seq_puts(m, " htw"); + if (cpu_has_xpa) + seq_puts(m, " xpa"); + if (cpu_has_loongson_mmi) + seq_puts(m, " loongson-mmi"); + if (cpu_has_loongson_cam) + seq_puts(m, " loongson-cam"); + if (cpu_has_loongson_ext) + seq_puts(m, " loongson-ext"); + if (cpu_has_loongson_ext2) + seq_puts(m, " loongson-ext2"); + seq_puts(m, "\n"); if (cpu_has_mmips) { seq_printf(m, "micromips kernel\t: %s\n", - (read_c0_config3() & MIPS_CONF3_ISA_OE) ? "yes" : "no"); + str_yes_no(read_c0_config3() & MIPS_CONF3_ISA_OE)); } + + seq_puts(m, "Options implemented\t:"); + if (cpu_has_tlb) + seq_puts(m, " tlb"); + if (cpu_has_ftlb) + seq_puts(m, " ftlb"); + if (cpu_has_tlbinv) + seq_puts(m, " tlbinv"); + if (cpu_has_segments) + seq_puts(m, " segments"); + if (cpu_has_rixiex) + seq_puts(m, " rixiex"); + if (cpu_has_ldpte) + seq_puts(m, " ldpte"); + if (cpu_has_maar) + seq_puts(m, " maar"); + if (cpu_has_rw_llb) + seq_puts(m, " rw_llb"); + if (cpu_has_4kex) + seq_puts(m, " 4kex"); + if (cpu_has_3k_cache) + seq_puts(m, " 3k_cache"); + if (cpu_has_4k_cache) + seq_puts(m, " 4k_cache"); + if (cpu_has_octeon_cache) + seq_puts(m, " octeon_cache"); + if (raw_cpu_has_fpu) + seq_puts(m, " fpu"); + if (cpu_has_32fpr) + seq_puts(m, " 32fpr"); + if (cpu_has_cache_cdex_p) + seq_puts(m, " cache_cdex_p"); + if (cpu_has_cache_cdex_s) + seq_puts(m, " cache_cdex_s"); + if (cpu_has_prefetch) + seq_puts(m, " prefetch"); + if (cpu_has_mcheck) + seq_puts(m, " mcheck"); + if (cpu_has_ejtag) + seq_puts(m, " ejtag"); + if (cpu_has_llsc) + seq_puts(m, " llsc"); + if (cpu_has_guestctl0ext) + seq_puts(m, " guestctl0ext"); + if (cpu_has_guestctl1) + seq_puts(m, " guestctl1"); + if (cpu_has_guestctl2) + seq_puts(m, " guestctl2"); + if (cpu_has_guestid) + seq_puts(m, " guestid"); + if (cpu_has_drg) + seq_puts(m, " drg"); + if (cpu_has_rixi) + seq_puts(m, " rixi"); + if (cpu_has_lpa) + seq_puts(m, " lpa"); + if (cpu_has_mvh) + seq_puts(m, " mvh"); + if (cpu_has_vtag_icache) + seq_puts(m, " vtag_icache"); + if (cpu_has_dc_aliases) + seq_puts(m, " dc_aliases"); + if (cpu_has_ic_fills_f_dc) + seq_puts(m, " ic_fills_f_dc"); + if (cpu_has_pindexed_dcache) + seq_puts(m, " pindexed_dcache"); + if (cpu_has_userlocal) + seq_puts(m, " userlocal"); + if (cpu_has_nofpuex) + seq_puts(m, " nofpuex"); + if (cpu_has_vint) + seq_puts(m, " vint"); + if (cpu_has_veic) + seq_puts(m, " veic"); + if (cpu_has_inclusive_pcaches) + seq_puts(m, " inclusive_pcaches"); + if (cpu_has_perf_cntr_intr_bit) + seq_puts(m, " perf_cntr_intr_bit"); + if (cpu_has_ufr) + seq_puts(m, " ufr"); + if (cpu_has_fre) + seq_puts(m, " fre"); + if (cpu_has_cdmm) + seq_puts(m, " cdmm"); + if (cpu_has_small_pages) + seq_puts(m, " small_pages"); + if (cpu_has_nan_legacy) + seq_puts(m, " nan_legacy"); + if (cpu_has_nan_2008) + seq_puts(m, " nan_2008"); + if (cpu_has_ebase_wg) + seq_puts(m, " ebase_wg"); + if (cpu_has_badinstr) + seq_puts(m, " badinstr"); + if (cpu_has_badinstrp) + seq_puts(m, " badinstrp"); + if (cpu_has_contextconfig) + seq_puts(m, " contextconfig"); + if (cpu_has_perf) + seq_puts(m, " perf"); + if (cpu_has_mac2008_only) + seq_puts(m, " mac2008_only"); + if (cpu_has_ftlbparex) + seq_puts(m, " ftlbparex"); + if (cpu_has_gsexcex) + seq_puts(m, " gsexcex"); + if (cpu_has_shared_ftlb_ram) + seq_puts(m, " shared_ftlb_ram"); + if (cpu_has_shared_ftlb_entries) + seq_puts(m, " shared_ftlb_entries"); + if (cpu_has_mipsmt_pertccounters) + seq_puts(m, " mipsmt_pertccounters"); + if (cpu_has_mmid) + seq_puts(m, " mmid"); + if (cpu_has_mm_sysad) + seq_puts(m, " mm_sysad"); + if (cpu_has_mm_full) + seq_puts(m, " mm_full"); + seq_puts(m, "\n"); + seq_printf(m, "shadow register sets\t: %d\n", cpu_data[n].srsets); seq_printf(m, "kscratch registers\t: %d\n", hweight8(cpu_data[n].kscratch_mask)); - seq_printf(m, "core\t\t\t: %d\n", cpu_data[n].core); + seq_printf(m, "package\t\t\t: %d\n", cpu_data[n].package); + seq_printf(m, "core\t\t\t: %d\n", cpu_core(&cpu_data[n])); + +#if defined(CONFIG_MIPS_MT_SMP) || defined(CONFIG_CPU_MIPSR6) + if (cpu_has_mipsmt) + seq_printf(m, "VPE\t\t\t: %d\n", cpu_vpe_id(&cpu_data[n])); + else if (cpu_has_vp) + seq_printf(m, "VP\t\t\t: %d\n", cpu_vpe_id(&cpu_data[n])); +#endif sprintf(fmt, "VCE%%c exceptions\t\t: %s\n", cpu_has_vce ? "%u" : "not available"); seq_printf(m, fmt, 'D', vced_count); seq_printf(m, fmt, 'I', vcei_count); - seq_printf(m, "\n"); + + proc_cpuinfo_notifier_args.m = m; + proc_cpuinfo_notifier_args.n = n; + + raw_notifier_call_chain(&proc_cpuinfo_chain, 0, + &proc_cpuinfo_notifier_args); + + seq_puts(m, "\n"); return 0; } @@ -121,7 +310,7 @@ static void *c_start(struct seq_file *m, loff_t *pos) { unsigned long i = *pos; - return i < NR_CPUS ? (void *) (i + 1) : NULL; + return i < nr_cpu_ids ? (void *) (i + 1) : NULL; } static void *c_next(struct seq_file *m, void *v, loff_t *pos) diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index ddc76103e78c..a3101f2268c6 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -9,45 +9,40 @@ * Copyright (C) 2004 Thiemo Seufer * Copyright (C) 2013 Imagination Technologies Ltd. */ +#include <linux/cpu.h> #include <linux/errno.h> -#include <linux/sched.h> -#include <linux/tick.h> -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/stddef.h> -#include <linux/unistd.h> -#include <linux/export.h> -#include <linux/ptrace.h> -#include <linux/mman.h> -#include <linux/personality.h> -#include <linux/sys.h> -#include <linux/user.h> #include <linux/init.h> -#include <linux/completion.h> #include <linux/kallsyms.h> +#include <linux/kernel.h> +#include <linux/nmi.h> +#include <linux/personality.h> +#include <linux/prctl.h> #include <linux/random.h> +#include <linux/sched.h> +#include <linux/sched/debug.h> +#include <linux/sched/task_stack.h> +#include <asm/abi.h> #include <asm/asm.h> -#include <asm/bootinfo.h> -#include <asm/cpu.h> +#include <asm/dsemul.h> #include <asm/dsp.h> +#include <asm/exec.h> #include <asm/fpu.h> -#include <asm/pgtable.h> +#include <asm/inst.h> +#include <asm/irq.h> +#include <asm/irq_regs.h> +#include <asm/isadep.h> +#include <asm/msa.h> +#include <asm/mips-cps.h> #include <asm/mipsregs.h> #include <asm/processor.h> -#include <asm/uaccess.h> -#include <asm/io.h> -#include <asm/elf.h> -#include <asm/isadep.h> -#include <asm/inst.h> +#include <asm/reg.h> #include <asm/stacktrace.h> #ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { - /* What the heck is this check doing ? */ - if (!cpu_isset(smp_processor_id(), cpu_callin_map)) - play_dead(); + play_dead(); } #endif @@ -59,62 +54,95 @@ void start_thread(struct pt_regs * regs, unsigned long pc, unsigned long sp) unsigned long status; /* New thread loses kernel privileges. */ - status = regs->cp0_status & ~(ST0_CU0|ST0_CU1|ST0_FR|KU_MASK); -#ifdef CONFIG_64BIT - status |= test_thread_flag(TIF_32BIT_REGS) ? 0 : ST0_FR; -#endif + status = regs->cp0_status & ~(ST0_CU0|ST0_CU1|ST0_CU2|ST0_FR|KU_MASK); status |= KU_USER; regs->cp0_status = status; + lose_fpu(0); + clear_thread_flag(TIF_MSA_CTX_LIVE); clear_used_math(); - clear_fpu_owner(); - if (cpu_has_dsp) - __init_dsp(); +#ifdef CONFIG_MIPS_FP_SUPPORT + atomic_set(¤t->thread.bd_emu_frame, BD_EMUFRAME_NONE); +#endif + init_dsp(); regs->cp0_epc = pc; regs->regs[29] = sp; } -void exit_thread(void) +void exit_thread(struct task_struct *tsk) { + /* + * User threads may have allocated a delay slot emulation frame. + * If so, clean up that allocation. + */ + if (!(current->flags & PF_KTHREAD)) + dsemul_thread_cleanup(tsk); } -void flush_thread(void) +int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { + /* + * Save any process state which is live in hardware registers to the + * parent context prior to duplication. This prevents the new child + * state becoming stale if the parent is preempted before copy_thread() + * gets a chance to save the parent's live hardware registers to the + * child context. + */ + preempt_disable(); + + if (is_msa_enabled()) + save_msa(current); + else if (is_fpu_owner()) + _save_fp(current); + + save_dsp(current); + + preempt_enable(); + + *dst = *src; + return 0; } -int copy_thread(unsigned long clone_flags, unsigned long usp, - unsigned long arg, struct task_struct *p) +/* + * Copy architecture-specific thread state + */ +int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) { + u64 clone_flags = args->flags; + unsigned long usp = args->stack; + unsigned long tls = args->tls; struct thread_info *ti = task_thread_info(p); struct pt_regs *childregs, *regs = current_pt_regs(); unsigned long childksp; - p->set_child_tid = p->clear_child_tid = NULL; childksp = (unsigned long)task_stack_page(p) + THREAD_SIZE - 32; - preempt_disable(); - - if (is_fpu_owner()) - save_fp(p); - - if (cpu_has_dsp) - save_dsp(p); - - preempt_enable(); - /* set up new TSS. */ childregs = (struct pt_regs *) childksp - 1; /* Put the stack after the struct pt_regs. */ childksp = (unsigned long) childregs; - p->thread.cp0_status = read_c0_status() & ~(ST0_CU2|ST0_CU1); - if (unlikely(p->flags & PF_KTHREAD)) { + p->thread.cp0_status = (read_c0_status() & ~(ST0_CU2|ST0_CU1)) | ST0_KERNEL_CUMASK; + + /* + * New tasks lose permission to use the fpu. This accelerates context + * switching for most programs since they don't use the fpu. + */ + clear_tsk_thread_flag(p, TIF_USEDFPU); + clear_tsk_thread_flag(p, TIF_USEDMSA); + clear_tsk_thread_flag(p, TIF_MSA_CTX_LIVE); + +#ifdef CONFIG_MIPS_MT_FPAFF + clear_tsk_thread_flag(p, TIF_FPUBOUND); +#endif /* CONFIG_MIPS_MT_FPAFF */ + + if (unlikely(args->fn)) { + /* kernel thread */ unsigned long status = p->thread.cp0_status; memset(childregs, 0, sizeof(struct pt_regs)); - ti->addr_limit = KERNEL_DS; - p->thread.reg16 = usp; /* fn */ - p->thread.reg17 = arg; + p->thread.reg16 = (unsigned long)args->fn; + p->thread.reg17 = (unsigned long)args->fn_arg; p->thread.reg29 = childksp; p->thread.reg31 = (unsigned long) ret_from_kernel_thread; -#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX) +#if defined(CONFIG_CPU_R3000) status = (status & ~(ST0_KUP | ST0_IEP | ST0_IEC)) | ((status & (ST0_KUC | ST0_IEC)) << 2); #else @@ -123,85 +151,30 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, childregs->cp0_status = status; return 0; } + + /* user thread */ *childregs = *regs; childregs->regs[7] = 0; /* Clear error flag */ childregs->regs[2] = 0; /* Child gets zero as return value */ if (usp) childregs->regs[29] = usp; - ti->addr_limit = USER_DS; p->thread.reg29 = (unsigned long) childregs; p->thread.reg31 = (unsigned long) ret_from_fork; - /* - * New tasks lose permission to use the fpu. This accelerates context - * switching for most programs since they don't use the fpu. - */ childregs->cp0_status &= ~(ST0_CU2|ST0_CU1); -#ifdef CONFIG_MIPS_MT_SMTC - /* - * SMTC restores TCStatus after Status, and the CU bits - * are aliased there. - */ - childregs->cp0_tcstatus &= ~(ST0_CU2|ST0_CU1); +#ifdef CONFIG_MIPS_FP_SUPPORT + atomic_set(&p->thread.bd_emu_frame, BD_EMUFRAME_NONE); #endif - clear_tsk_thread_flag(p, TIF_USEDFPU); - -#ifdef CONFIG_MIPS_MT_FPAFF - clear_tsk_thread_flag(p, TIF_FPUBOUND); -#endif /* CONFIG_MIPS_MT_FPAFF */ if (clone_flags & CLONE_SETTLS) - ti->tp_value = regs->regs[7]; + ti->tp_value = tls; return 0; } -/* Fill in the fpu structure for a core dump.. */ -int dump_fpu(struct pt_regs *regs, elf_fpregset_t *r) -{ - memcpy(r, ¤t->thread.fpu, sizeof(current->thread.fpu)); - - return 1; -} - -void elf_dump_regs(elf_greg_t *gp, struct pt_regs *regs) -{ - int i; - - for (i = 0; i < EF_R0; i++) - gp[i] = 0; - gp[EF_R0] = 0; - for (i = 1; i <= 31; i++) - gp[EF_R0 + i] = regs->regs[i]; - gp[EF_R26] = 0; - gp[EF_R27] = 0; - gp[EF_LO] = regs->lo; - gp[EF_HI] = regs->hi; - gp[EF_CP0_EPC] = regs->cp0_epc; - gp[EF_CP0_BADVADDR] = regs->cp0_badvaddr; - gp[EF_CP0_STATUS] = regs->cp0_status; - gp[EF_CP0_CAUSE] = regs->cp0_cause; -#ifdef EF_UNUSED0 - gp[EF_UNUSED0] = 0; -#endif -} - -int dump_task_regs(struct task_struct *tsk, elf_gregset_t *regs) -{ - elf_dump_regs(*regs, task_pt_regs(tsk)); - return 1; -} - -int dump_task_fpu(struct task_struct *t, elf_fpregset_t *fpr) -{ - memcpy(fpr, &t->thread.fpu, sizeof(current->thread.fpu)); - - return 1; -} - -#ifdef CONFIG_CC_STACKPROTECTOR +#ifdef CONFIG_STACKPROTECTOR #include <linux/stackprotector.h> unsigned long __stack_chk_guard __read_mostly; EXPORT_SYMBOL(__stack_chk_guard); @@ -217,11 +190,39 @@ struct mips_frame_info { #define J_TARGET(pc,target) \ (((unsigned long)(pc) & 0xf0000000) | ((target) << 2)) -static inline int is_ra_save_ins(union mips_instruction *ip) +static inline int is_jr_ra_ins(union mips_instruction *ip) { #ifdef CONFIG_CPU_MICROMIPS - union mips_instruction mmi; + /* + * jr16 ra + * jr ra + */ + if (mm_insn_16bit(ip->word >> 16)) { + if (ip->mm16_r5_format.opcode == mm_pool16c_op && + ip->mm16_r5_format.rt == mm_jr16_op && + ip->mm16_r5_format.imm == 31) + return 1; + return 0; + } + + if (ip->r_format.opcode == mm_pool32a_op && + ip->r_format.func == mm_pool32axf_op && + ((ip->u_format.uimmediate >> 6) & GENMASK(9, 0)) == mm_jalr_op && + ip->r_format.rt == 31) + return 1; + return 0; +#else + if (ip->r_format.opcode == spec_op && + ip->r_format.func == jr_op && + ip->r_format.rs == 31) + return 1; + return 0; +#endif +} +static inline int is_ra_save_ins(union mips_instruction *ip, int *poff) +{ +#ifdef CONFIG_CPU_MICROMIPS /* * swsp ra,offset * swm16 reglist,offset(sp) @@ -231,29 +232,85 @@ static inline int is_ra_save_ins(union mips_instruction *ip) * * microMIPS is way more fun... */ - if (mm_insn_16bit(ip->halfword[0])) { - mmi.word = (ip->halfword[0] << 16); - return ((mmi.mm16_r5_format.opcode == mm_swsp16_op && - mmi.mm16_r5_format.rt == 31) || - (mmi.mm16_m_format.opcode == mm_pool16c_op && - mmi.mm16_m_format.func == mm_swm16_op)); + if (mm_insn_16bit(ip->word >> 16)) { + switch (ip->mm16_r5_format.opcode) { + case mm_swsp16_op: + if (ip->mm16_r5_format.rt != 31) + return 0; + + *poff = ip->mm16_r5_format.imm; + *poff = (*poff << 2) / sizeof(ulong); + return 1; + + case mm_pool16c_op: + switch (ip->mm16_m_format.func) { + case mm_swm16_op: + *poff = ip->mm16_m_format.imm; + *poff += 1 + ip->mm16_m_format.rlist; + *poff = (*poff << 2) / sizeof(ulong); + return 1; + + default: + return 0; + } + + default: + return 0; + } } - else { - mmi.halfword[0] = ip->halfword[1]; - mmi.halfword[1] = ip->halfword[0]; - return ((mmi.mm_m_format.opcode == mm_pool32b_op && - mmi.mm_m_format.rd > 9 && - mmi.mm_m_format.base == 29 && - mmi.mm_m_format.func == mm_swm32_func) || - (mmi.i_format.opcode == mm_sw32_op && - mmi.i_format.rs == 29 && - mmi.i_format.rt == 31)); + + switch (ip->i_format.opcode) { + case mm_sw32_op: + if (ip->i_format.rs != 29) + return 0; + if (ip->i_format.rt != 31) + return 0; + + *poff = ip->i_format.simmediate / sizeof(ulong); + return 1; + + case mm_pool32b_op: + switch (ip->mm_m_format.func) { + case mm_swm32_func: + if (ip->mm_m_format.rd < 0x10) + return 0; + if (ip->mm_m_format.base != 29) + return 0; + + *poff = ip->mm_m_format.simmediate; + *poff += (ip->mm_m_format.rd & 0xf) * sizeof(u32); + *poff /= sizeof(ulong); + return 1; + default: + return 0; + } + + default: + return 0; } #else /* sw / sd $ra, offset($sp) */ - return (ip->i_format.opcode == sw_op || ip->i_format.opcode == sd_op) && - ip->i_format.rs == 29 && - ip->i_format.rt == 31; + if ((ip->i_format.opcode == sw_op || ip->i_format.opcode == sd_op) && + ip->i_format.rs == 29 && ip->i_format.rt == 31) { + *poff = ip->i_format.simmediate / sizeof(ulong); + return 1; + } +#ifdef CONFIG_CPU_LOONGSON64 + if ((ip->loongson3_lswc2_format.opcode == swc2_op) && + (ip->loongson3_lswc2_format.ls == 1) && + (ip->loongson3_lswc2_format.fr == 0) && + (ip->loongson3_lswc2_format.base == 29)) { + if (ip->loongson3_lswc2_format.rt == 31) { + *poff = ip->loongson3_lswc2_format.offset << 1; + return 1; + } + if (ip->loongson3_lswc2_format.rq == 31) { + *poff = (ip->loongson3_lswc2_format.offset << 1) + 1; + return 1; + } + } +#endif + return 0; #endif } @@ -268,18 +325,21 @@ static inline int is_jump_ins(union mips_instruction *ip) * * microMIPS is kind of more fun... */ - union mips_instruction mmi; - - mmi.word = (ip->halfword[0] << 16); + if (mm_insn_16bit(ip->word >> 16)) { + if ((ip->mm16_r5_format.opcode == mm_pool16c_op && + (ip->mm16_r5_format.rt & mm_jr16_op) == mm_jr16_op)) + return 1; + return 0; + } - if ((mmi.mm16_r5_format.opcode == mm_pool16c_op && - (mmi.mm16_r5_format.rt & mm_jr16_op) == mm_jr16_op) || - ip->j_format.opcode == mm_jal32_op) + if (ip->j_format.opcode == mm_j32_op) + return 1; + if (ip->j_format.opcode == mm_jal32_op) return 1; if (ip->r_format.opcode != mm_pool32a_op || ip->r_format.func != mm_pool32axf_op) return 0; - return (((ip->u_format.uimmediate >> 6) & mm_jalr_op) == mm_jalr_op); + return ((ip->u_format.uimmediate >> 6) & mm_jalr_op) == mm_jalr_op; #else if (ip->j_format.opcode == j_op) return 1; @@ -291,9 +351,11 @@ static inline int is_jump_ins(union mips_instruction *ip) #endif } -static inline int is_sp_move_ins(union mips_instruction *ip) +static inline int is_sp_move_ins(union mips_instruction *ip, int *frame_size) { #ifdef CONFIG_CPU_MICROMIPS + unsigned short tmp; + /* * addiusp -imm * addius5 sp,-imm @@ -302,86 +364,107 @@ static inline int is_sp_move_ins(union mips_instruction *ip) * * microMIPS is not more fun... */ - if (mm_insn_16bit(ip->halfword[0])) { - union mips_instruction mmi; - - mmi.word = (ip->halfword[0] << 16); - return ((mmi.mm16_r3_format.opcode == mm_pool16d_op && - mmi.mm16_r3_format.simmediate && mm_addiusp_func) || - (mmi.mm16_r5_format.opcode == mm_pool16d_op && - mmi.mm16_r5_format.rt == 29)); + if (mm_insn_16bit(ip->word >> 16)) { + if (ip->mm16_r3_format.opcode == mm_pool16d_op && + ip->mm16_r3_format.simmediate & mm_addiusp_func) { + tmp = ip->mm_b0_format.simmediate >> 1; + tmp = ((tmp & 0x1ff) ^ 0x100) - 0x100; + if ((tmp + 2) < 4) /* 0x0,0x1,0x1fe,0x1ff are special */ + tmp ^= 0x100; + *frame_size = -(signed short)(tmp << 2); + return 1; + } + if (ip->mm16_r5_format.opcode == mm_pool16d_op && + ip->mm16_r5_format.rt == 29) { + tmp = ip->mm16_r5_format.imm >> 1; + *frame_size = -(signed short)(tmp & 0xf); + return 1; + } + return 0; + } + + if (ip->mm_i_format.opcode == mm_addiu32_op && + ip->mm_i_format.rt == 29 && ip->mm_i_format.rs == 29) { + *frame_size = -ip->i_format.simmediate; + return 1; } - return (ip->mm_i_format.opcode == mm_addiu32_op && - ip->mm_i_format.rt == 29 && ip->mm_i_format.rs == 29); #else /* addiu/daddiu sp,sp,-imm */ if (ip->i_format.rs != 29 || ip->i_format.rt != 29) return 0; - if (ip->i_format.opcode == addiu_op || ip->i_format.opcode == daddiu_op) + + if (ip->i_format.opcode == addiu_op || + ip->i_format.opcode == daddiu_op) { + *frame_size = -ip->i_format.simmediate; return 1; + } #endif return 0; } static int get_frame_info(struct mips_frame_info *info) { -#ifdef CONFIG_CPU_MICROMIPS - union mips_instruction *ip = (void *) (((char *) info->func) - 1); -#else - union mips_instruction *ip = info->func; -#endif - unsigned max_insns = info->func_size / sizeof(union mips_instruction); - unsigned i; + bool is_mmips = IS_ENABLED(CONFIG_CPU_MICROMIPS); + union mips_instruction insn, *ip, *ip_end; + unsigned int last_insn_size = 0; + bool saw_jump = false; info->pc_offset = -1; info->frame_size = 0; + ip = (void *)msk_isa16_mode((ulong)info->func); if (!ip) goto err; - if (max_insns == 0) - max_insns = 128U; /* unknown function size */ - max_insns = min(128U, max_insns); + ip_end = (void *)ip + (info->func_size ? info->func_size : 512); - for (i = 0; i < max_insns; i++, ip++) { + while (ip < ip_end) { + ip = (void *)ip + last_insn_size; + + if (is_mmips && mm_insn_16bit(ip->halfword[0])) { + insn.word = ip->halfword[0] << 16; + last_insn_size = 2; + } else if (is_mmips) { + insn.word = ip->halfword[0] << 16 | ip->halfword[1]; + last_insn_size = 4; + } else { + insn.word = ip->word; + last_insn_size = 4; + } - if (is_jump_ins(ip)) + if (is_jr_ra_ins(ip)) { break; - if (!info->frame_size) { - if (is_sp_move_ins(ip)) - { -#ifdef CONFIG_CPU_MICROMIPS - if (mm_insn_16bit(ip->halfword[0])) - { - unsigned short tmp; - - if (ip->halfword[0] & mm_addiusp_func) - { - tmp = (((ip->halfword[0] >> 1) & 0x1ff) << 2); - info->frame_size = -(signed short)(tmp | ((tmp & 0x100) ? 0xfe00 : 0)); - } else { - tmp = (ip->halfword[0] >> 1); - info->frame_size = -(signed short)(tmp & 0xf); - } - ip = (void *) &ip->halfword[1]; - ip--; - } else -#endif - info->frame_size = - ip->i_format.simmediate; - } + } else if (!info->frame_size) { + is_sp_move_ins(&insn, &info->frame_size); + continue; + } else if (!saw_jump && is_jump_ins(ip)) { + /* + * If we see a jump instruction, we are finished + * with the frame save. + * + * Some functions can have a shortcut return at + * the beginning of the function, so don't start + * looking for jump instruction until we see the + * frame setup. + * + * The RA save instruction can get put into the + * delay slot of the jump instruction, so look + * at the next instruction, too. + */ + saw_jump = true; continue; } - if (info->pc_offset == -1 && is_ra_save_ins(ip)) { - info->pc_offset = - ip->i_format.simmediate / sizeof(long); + if (info->pc_offset == -1 && + is_ra_save_ins(&insn, &info->pc_offset)) + break; + if (saw_jump) break; - } } if (info->frame_size && info->pc_offset >= 0) /* nested */ return 0; if (info->pc_offset < 0) /* leaf */ return 1; - /* prologue seems boggus... */ + /* prologue seems bogus... */ err: return -1; } @@ -430,7 +513,7 @@ static int __init frame_info_init(void) /* * Without schedule() frame info, result given by - * thread_saved_pc() and get_wchan() are not reliable. + * thread_saved_pc() and __get_wchan() are not reliable. */ if (schedule_mfi.pc_offset < 0) printk("Can't analyze schedule() prologue at %p\n", schedule); @@ -443,7 +526,7 @@ arch_initcall(frame_info_init); /* * Return saved PC of a blocked thread. */ -unsigned long thread_saved_pc(struct task_struct *tsk) +static unsigned long thread_saved_pc(struct task_struct *tsk) { struct thread_struct *t = &tsk->thread; @@ -463,31 +546,52 @@ unsigned long notrace unwind_stack_by_address(unsigned long stack_page, unsigned long pc, unsigned long *ra) { + unsigned long low, high, irq_stack_high; struct mips_frame_info info; unsigned long size, ofs; + struct pt_regs *regs; int leaf; - extern void ret_from_irq(void); - extern void ret_from_exception(void); if (!stack_page) return 0; /* - * If we reached the bottom of interrupt context, - * return saved pc in pt_regs. + * IRQ stacks start at IRQ_STACK_START + * task stacks at THREAD_SIZE - 32 */ - if (pc == (unsigned long)ret_from_irq || - pc == (unsigned long)ret_from_exception) { - struct pt_regs *regs; - if (*sp >= stack_page && - *sp + sizeof(*regs) <= stack_page + THREAD_SIZE - 32) { - regs = (struct pt_regs *)*sp; - pc = regs->cp0_epc; - if (__kernel_text_address(pc)) { - *sp = regs->regs[29]; - *ra = regs->regs[31]; - return pc; - } + low = stack_page; + if (!preemptible() && on_irq_stack(raw_smp_processor_id(), *sp)) { + high = stack_page + IRQ_STACK_START; + irq_stack_high = high; + } else { + high = stack_page + THREAD_SIZE - 32; + irq_stack_high = 0; + } + + /* + * If we reached the top of the interrupt stack, start unwinding + * the interrupted task stack. + */ + if (unlikely(*sp == irq_stack_high)) { + unsigned long task_sp = *(unsigned long *)*sp; + + /* + * Check that the pointer saved in the IRQ stack head points to + * something within the stack of the current task + */ + if (!object_is_on_stack((void *)task_sp)) + return 0; + + /* + * Follow pointer to tasks kernel stack frame where interrupted + * state was saved. + */ + regs = (struct pt_regs *)task_sp; + pc = regs->cp0_epc; + if (!user_mode(regs) && __kernel_text_address(pc)) { + *sp = regs->regs[29]; + *ra = regs->regs[31]; + return pc; } return 0; } @@ -508,8 +612,7 @@ unsigned long notrace unwind_stack_by_address(unsigned long stack_page, if (leaf < 0) return 0; - if (*sp < stack_page || - *sp + info.frame_size > stack_page + THREAD_SIZE - 32) + if (*sp < low || *sp + info.frame_size > high) return 0; if (leaf) @@ -533,15 +636,27 @@ EXPORT_SYMBOL(unwind_stack_by_address); unsigned long unwind_stack(struct task_struct *task, unsigned long *sp, unsigned long pc, unsigned long *ra) { - unsigned long stack_page = (unsigned long)task_stack_page(task); + unsigned long stack_page = 0; + int cpu; + + for_each_possible_cpu(cpu) { + if (on_irq_stack(cpu, *sp)) { + stack_page = (unsigned long)irq_stack[cpu]; + break; + } + } + + if (!stack_page) + stack_page = (unsigned long)task_stack_page(task); + return unwind_stack_by_address(stack_page, sp, pc, ra); } #endif /* - * get_wchan - a maintenance nightmare^W^Wpain in the ass ... + * __get_wchan - a maintenance nightmare^W^Wpain in the ass ... */ -unsigned long get_wchan(struct task_struct *task) +unsigned long __get_wchan(struct task_struct *task) { unsigned long pc = 0; #ifdef CONFIG_KALLSYMS @@ -549,8 +664,6 @@ unsigned long get_wchan(struct task_struct *task) unsigned long ra = 0; #endif - if (!task || task == current || task->state == TASK_RUNNING) - goto out; if (!task_stack_page(task)) goto out; @@ -567,6 +680,33 @@ out: return pc; } +unsigned long mips_stack_top(void) +{ + unsigned long top = TASK_SIZE & PAGE_MASK; + + if (IS_ENABLED(CONFIG_MIPS_FP_SUPPORT)) { + /* One page for branch delay slot "emulation" */ + top -= PAGE_SIZE; + } + + /* Space for the VDSO, data page & GIC user page */ + if (current->thread.abi) { + top -= PAGE_ALIGN(current->thread.abi->vdso->size); + top -= VDSO_NR_PAGES * PAGE_SIZE; + top -= mips_gic_present() ? PAGE_SIZE : 0; + + /* Space to randomize the VDSO base */ + if (current->flags & PF_RANDOMIZE) + top -= VDSO_RANDOMIZE_SIZE; + } + + /* Space for cache colour alignment */ + if (cpu_has_dc_aliases) + top -= shm_align_mask + 1; + + return top; +} + /* * Don't forget that the stack pointer must be aligned on a 8 bytes * boundary for 32-bits ABI and 16 bytes for 64-bits ABI. @@ -574,7 +714,201 @@ out: unsigned long arch_align_stack(unsigned long sp) { if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) - sp -= get_random_int() & ~PAGE_MASK; + sp -= get_random_u32_below(PAGE_SIZE); return sp & ALMASK; } + +static struct cpumask backtrace_csd_busy; + +static void handle_backtrace(void *info) +{ + nmi_cpu_backtrace(get_irq_regs()); + cpumask_clear_cpu(smp_processor_id(), &backtrace_csd_busy); +} + +static DEFINE_PER_CPU(call_single_data_t, backtrace_csd) = + CSD_INIT(handle_backtrace, NULL); + +static void raise_backtrace(cpumask_t *mask) +{ + call_single_data_t *csd; + int cpu; + + for_each_cpu(cpu, mask) { + /* + * If we previously sent an IPI to the target CPU & it hasn't + * cleared its bit in the busy cpumask then it didn't handle + * our previous IPI & it's not safe for us to reuse the + * call_single_data_t. + */ + if (cpumask_test_and_set_cpu(cpu, &backtrace_csd_busy)) { + pr_warn("Unable to send backtrace IPI to CPU%u - perhaps it hung?\n", + cpu); + continue; + } + + csd = &per_cpu(backtrace_csd, cpu); + smp_call_function_single_async(cpu, csd); + } +} + +void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) +{ + nmi_trigger_cpumask_backtrace(mask, exclude_cpu, raise_backtrace); +} + +int mips_get_process_fp_mode(struct task_struct *task) +{ + int value = 0; + + if (!test_tsk_thread_flag(task, TIF_32BIT_FPREGS)) + value |= PR_FP_MODE_FR; + if (test_tsk_thread_flag(task, TIF_HYBRID_FPREGS)) + value |= PR_FP_MODE_FRE; + + return value; +} + +static long prepare_for_fp_mode_switch(void *unused) +{ + /* + * This is icky, but we use this to simply ensure that all CPUs have + * context switched, regardless of whether they were previously running + * kernel or user code. This ensures that no CPU that a mode-switching + * program may execute on keeps its FPU enabled (& in the old mode) + * throughout the mode switch. + */ + return 0; +} + +int mips_set_process_fp_mode(struct task_struct *task, unsigned int value) +{ + const unsigned int known_bits = PR_FP_MODE_FR | PR_FP_MODE_FRE; + struct task_struct *t; + struct cpumask process_cpus; + int cpu; + + /* If nothing to change, return right away, successfully. */ + if (value == mips_get_process_fp_mode(task)) + return 0; + + /* Only accept a mode change if 64-bit FP enabled for o32. */ + if (!IS_ENABLED(CONFIG_MIPS_O32_FP64_SUPPORT)) + return -EOPNOTSUPP; + + /* And only for o32 tasks. */ + if (IS_ENABLED(CONFIG_64BIT) && !test_thread_flag(TIF_32BIT_REGS)) + return -EOPNOTSUPP; + + /* Check the value is valid */ + if (value & ~known_bits) + return -EOPNOTSUPP; + + /* Setting FRE without FR is not supported. */ + if ((value & (PR_FP_MODE_FR | PR_FP_MODE_FRE)) == PR_FP_MODE_FRE) + return -EOPNOTSUPP; + + /* Avoid inadvertently triggering emulation */ + if ((value & PR_FP_MODE_FR) && raw_cpu_has_fpu && + !(raw_current_cpu_data.fpu_id & MIPS_FPIR_F64)) + return -EOPNOTSUPP; + if ((value & PR_FP_MODE_FRE) && raw_cpu_has_fpu && !cpu_has_fre) + return -EOPNOTSUPP; + + /* FR = 0 not supported in MIPS R6 */ + if (!(value & PR_FP_MODE_FR) && raw_cpu_has_fpu && cpu_has_mips_r6) + return -EOPNOTSUPP; + + /* Indicate the new FP mode in each thread */ + for_each_thread(task, t) { + /* Update desired FP register width */ + if (value & PR_FP_MODE_FR) { + clear_tsk_thread_flag(t, TIF_32BIT_FPREGS); + } else { + set_tsk_thread_flag(t, TIF_32BIT_FPREGS); + clear_tsk_thread_flag(t, TIF_MSA_CTX_LIVE); + } + + /* Update desired FP single layout */ + if (value & PR_FP_MODE_FRE) + set_tsk_thread_flag(t, TIF_HYBRID_FPREGS); + else + clear_tsk_thread_flag(t, TIF_HYBRID_FPREGS); + } + + /* + * We need to ensure that all threads in the process have switched mode + * before returning, in order to allow userland to not worry about + * races. We can do this by forcing all CPUs that any thread in the + * process may be running on to schedule something else - in this case + * prepare_for_fp_mode_switch(). + * + * We begin by generating a mask of all CPUs that any thread in the + * process may be running on. + */ + cpumask_clear(&process_cpus); + for_each_thread(task, t) + cpumask_set_cpu(task_cpu(t), &process_cpus); + + /* + * Now we schedule prepare_for_fp_mode_switch() on each of those CPUs. + * + * The CPUs may have rescheduled already since we switched mode or + * generated the cpumask, but that doesn't matter. If the task in this + * process is scheduled out then our scheduling + * prepare_for_fp_mode_switch() will simply be redundant. If it's + * scheduled in then it will already have picked up the new FP mode + * whilst doing so. + */ + cpus_read_lock(); + for_each_cpu_and(cpu, &process_cpus, cpu_online_mask) + work_on_cpu(cpu, prepare_for_fp_mode_switch, NULL); + cpus_read_unlock(); + + return 0; +} + +#if defined(CONFIG_32BIT) || defined(CONFIG_MIPS32_O32) +void mips_dump_regs32(u32 *uregs, const struct pt_regs *regs) +{ + unsigned int i; + + for (i = MIPS32_EF_R1; i <= MIPS32_EF_R31; i++) { + /* k0/k1 are copied as zero. */ + if (i == MIPS32_EF_R26 || i == MIPS32_EF_R27) + uregs[i] = 0; + else + uregs[i] = regs->regs[i - MIPS32_EF_R0]; + } + + uregs[MIPS32_EF_LO] = regs->lo; + uregs[MIPS32_EF_HI] = regs->hi; + uregs[MIPS32_EF_CP0_EPC] = regs->cp0_epc; + uregs[MIPS32_EF_CP0_BADVADDR] = regs->cp0_badvaddr; + uregs[MIPS32_EF_CP0_STATUS] = regs->cp0_status; + uregs[MIPS32_EF_CP0_CAUSE] = regs->cp0_cause; +} +#endif /* CONFIG_32BIT || CONFIG_MIPS32_O32 */ + +#ifdef CONFIG_64BIT +void mips_dump_regs64(u64 *uregs, const struct pt_regs *regs) +{ + unsigned int i; + + for (i = MIPS64_EF_R1; i <= MIPS64_EF_R31; i++) { + /* k0/k1 are copied as zero. */ + if (i == MIPS64_EF_R26 || i == MIPS64_EF_R27) + uregs[i] = 0; + else + uregs[i] = regs->regs[i - MIPS64_EF_R0]; + } + + uregs[MIPS64_EF_LO] = regs->lo; + uregs[MIPS64_EF_HI] = regs->hi; + uregs[MIPS64_EF_CP0_EPC] = regs->cp0_epc; + uregs[MIPS64_EF_CP0_BADVADDR] = regs->cp0_badvaddr; + uregs[MIPS64_EF_CP0_STATUS] = regs->cp0_status; + uregs[MIPS64_EF_CP0_CAUSE] = regs->cp0_cause; +} +#endif /* CONFIG_64BIT */ diff --git a/arch/mips/kernel/prom.c b/arch/mips/kernel/prom.c index 7e954042f252..4fd6da0a06c3 100644 --- a/arch/mips/kernel/prom.c +++ b/arch/mips/kernel/prom.c @@ -1,25 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * MIPS support for CONFIG_OF device tree support * * Copyright (C) 2010 Cisco Systems Inc. <dediao@cisco.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include <linux/init.h> #include <linux/export.h> #include <linux/errno.h> #include <linux/types.h> -#include <linux/bootmem.h> -#include <linux/initrd.h> +#include <linux/memblock.h> #include <linux/debugfs.h> #include <linux/of.h> #include <linux/of_fdt.h> -#include <linux/of_irq.h> #include <linux/of_platform.h> +#include <asm/bootinfo.h> #include <asm/page.h> #include <asm/prom.h> @@ -30,8 +26,10 @@ __init void mips_set_machine_name(const char *name) if (name == NULL) return; - strlcpy(mips_machine_name, name, sizeof(mips_machine_name)); + strscpy(mips_machine_name, name, sizeof(mips_machine_name)); pr_info("MIPS: machine is %s\n", mips_get_machine_name()); + + dump_stack_set_arch_desc(name); } char *mips_get_machine_name(void) @@ -39,76 +37,38 @@ char *mips_get_machine_name(void) return mips_machine_name; } -#ifdef CONFIG_OF -int __init early_init_dt_scan_memory_arch(unsigned long node, - const char *uname, int depth, - void *data) -{ - return early_init_dt_scan_memory(node, uname, depth, data); -} +#ifdef CONFIG_USE_OF -void __init early_init_dt_add_memory_arch(u64 base, u64 size) +void __init __dt_setup_arch(void *bph) { - return add_memory_region(base, size, BOOT_MEM_RAM); -} + if (!early_init_dt_scan(bph, __pa(bph))) + return; -void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align) -{ - return __alloc_bootmem(size, align, __pa(MAX_DMA_ADDRESS)); + mips_set_machine_name(of_flat_dt_get_machine_name()); } -#ifdef CONFIG_BLK_DEV_INITRD -void __init early_init_dt_setup_initrd_arch(unsigned long start, - unsigned long end) +int __init __dt_register_buses(const char *bus0, const char *bus1) { - initrd_start = (unsigned long)__va(start); - initrd_end = (unsigned long)__va(end); - initrd_below_start_ok = 1; -} -#endif + static struct of_device_id of_ids[3]; -int __init early_init_dt_scan_model(unsigned long node, const char *uname, - int depth, void *data) -{ - if (!depth) { - char *model = of_get_flat_dt_prop(node, "model", NULL); + if (!of_have_populated_dt()) + panic("device tree not present"); - if (model) - mips_set_machine_name(model); + strscpy(of_ids[0].compatible, bus0, sizeof(of_ids[0].compatible)); + if (bus1) { + strscpy(of_ids[1].compatible, bus1, + sizeof(of_ids[1].compatible)); } - return 0; -} - -void __init early_init_devtree(void *params) -{ - /* Setup flat device-tree pointer */ - initial_boot_params = params; - - /* Retrieve various informations from the /chosen node of the - * device-tree, including the platform type, initrd location and - * size, and more ... - */ - of_scan_flat_dt(early_init_dt_scan_chosen, arcs_cmdline); + if (of_platform_populate(NULL, of_ids, NULL, NULL)) + panic("failed to populate DT"); - /* Scan memory nodes */ - of_scan_flat_dt(early_init_dt_scan_root, NULL); - of_scan_flat_dt(early_init_dt_scan_memory_arch, NULL); - - /* try to load the mips machine name */ - of_scan_flat_dt(early_init_dt_scan_model, NULL); + return 0; } -void __init __dt_setup_arch(struct boot_param_header *bph) +void __weak __init device_tree_init(void) { - if (be32_to_cpu(bph->magic) != OF_DT_HEADER) { - pr_err("DTB has bad magic, ignoring builtin OF DTB\n"); - - return; - } - - initial_boot_params = bph; - - early_init_devtree(initial_boot_params); + unflatten_and_copy_device_tree(); } + #endif diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c index 8ae1ebef8b71..3f4c94c88124 100644 --- a/arch/mips/kernel/ptrace.c +++ b/arch/mips/kernel/ptrace.c @@ -16,29 +16,45 @@ */ #include <linux/compiler.h> #include <linux/context_tracking.h> +#include <linux/elf.h> #include <linux/kernel.h> #include <linux/sched.h> +#include <linux/sched/task_stack.h> #include <linux/mm.h> #include <linux/errno.h> #include <linux/ptrace.h> +#include <linux/regset.h> #include <linux/smp.h> -#include <linux/user.h> #include <linux/security.h> +#include <linux/stddef.h> #include <linux/audit.h> #include <linux/seccomp.h> +#include <linux/ftrace.h> +#include <asm/branch.h> #include <asm/byteorder.h> #include <asm/cpu.h> +#include <asm/cpu-info.h> #include <asm/dsp.h> #include <asm/fpu.h> #include <asm/mipsregs.h> #include <asm/mipsmtregs.h> -#include <asm/pgtable.h> #include <asm/page.h> -#include <asm/uaccess.h> +#include <asm/processor.h> +#include <asm/syscall.h> +#include <linux/uaccess.h> #include <asm/bootinfo.h> #include <asm/reg.h> +#define CREATE_TRACE_POINTS +#include <trace/events/syscalls.h> + +unsigned long exception_ip(struct pt_regs *regs) +{ + return exception_epc(regs); +} +EXPORT_SYMBOL(exception_ip); + /* * Called by kernel/ptrace.c when detaching.. * @@ -55,24 +71,24 @@ void ptrace_disable(struct task_struct *child) * for 32-bit kernels and for 32-bit processes on a 64-bit kernel. * Registers are sign extended to fill the available space. */ -int ptrace_getregs(struct task_struct *child, __s64 __user *data) +int ptrace_getregs(struct task_struct *child, struct user_pt_regs __user *data) { struct pt_regs *regs; int i; - if (!access_ok(VERIFY_WRITE, data, 38 * 8)) + if (!access_ok(data, 38 * 8)) return -EIO; regs = task_pt_regs(child); for (i = 0; i < 32; i++) - __put_user((long)regs->regs[i], data + i); - __put_user((long)regs->lo, data + EF_LO - EF_R0); - __put_user((long)regs->hi, data + EF_HI - EF_R0); - __put_user((long)regs->cp0_epc, data + EF_CP0_EPC - EF_R0); - __put_user((long)regs->cp0_badvaddr, data + EF_CP0_BADVADDR - EF_R0); - __put_user((long)regs->cp0_status, data + EF_CP0_STATUS - EF_R0); - __put_user((long)regs->cp0_cause, data + EF_CP0_CAUSE - EF_R0); + __put_user((long)regs->regs[i], (__s64 __user *)&data->regs[i]); + __put_user((long)regs->lo, (__s64 __user *)&data->lo); + __put_user((long)regs->hi, (__s64 __user *)&data->hi); + __put_user((long)regs->cp0_epc, (__s64 __user *)&data->cp0_epc); + __put_user((long)regs->cp0_badvaddr, (__s64 __user *)&data->cp0_badvaddr); + __put_user((long)regs->cp0_status, (__s64 __user *)&data->cp0_status); + __put_user((long)regs->cp0_cause, (__s64 __user *)&data->cp0_cause); return 0; } @@ -82,88 +98,26 @@ int ptrace_getregs(struct task_struct *child, __s64 __user *data) * the 64-bit format. On a 32-bit kernel only the lower order half * (according to endianness) will be used. */ -int ptrace_setregs(struct task_struct *child, __s64 __user *data) +int ptrace_setregs(struct task_struct *child, struct user_pt_regs __user *data) { struct pt_regs *regs; int i; - if (!access_ok(VERIFY_READ, data, 38 * 8)) + if (!access_ok(data, 38 * 8)) return -EIO; regs = task_pt_regs(child); for (i = 0; i < 32; i++) - __get_user(regs->regs[i], data + i); - __get_user(regs->lo, data + EF_LO - EF_R0); - __get_user(regs->hi, data + EF_HI - EF_R0); - __get_user(regs->cp0_epc, data + EF_CP0_EPC - EF_R0); + __get_user(regs->regs[i], (__s64 __user *)&data->regs[i]); + __get_user(regs->lo, (__s64 __user *)&data->lo); + __get_user(regs->hi, (__s64 __user *)&data->hi); + __get_user(regs->cp0_epc, (__s64 __user *)&data->cp0_epc); /* badvaddr, status, and cause may not be written. */ - return 0; -} - -int ptrace_getfpregs(struct task_struct *child, __u32 __user *data) -{ - int i; - unsigned int tmp; - - if (!access_ok(VERIFY_WRITE, data, 33 * 8)) - return -EIO; - - if (tsk_used_math(child)) { - fpureg_t *fregs = get_fpu_regs(child); - for (i = 0; i < 32; i++) - __put_user(fregs[i], i + (__u64 __user *) data); - } else { - for (i = 0; i < 32; i++) - __put_user((__u64) -1, i + (__u64 __user *) data); - } - - __put_user(child->thread.fpu.fcr31, data + 64); - - preempt_disable(); - if (cpu_has_fpu) { - unsigned int flags; - - if (cpu_has_mipsmt) { - unsigned int vpflags = dvpe(); - flags = read_c0_status(); - __enable_fpu(); - __asm__ __volatile__("cfc1\t%0,$0" : "=r" (tmp)); - write_c0_status(flags); - evpe(vpflags); - } else { - flags = read_c0_status(); - __enable_fpu(); - __asm__ __volatile__("cfc1\t%0,$0" : "=r" (tmp)); - write_c0_status(flags); - } - } else { - tmp = 0; - } - preempt_enable(); - __put_user(tmp, data + 65); - - return 0; -} - -int ptrace_setfpregs(struct task_struct *child, __u32 __user *data) -{ - fpureg_t *fregs; - int i; - - if (!access_ok(VERIFY_READ, data, 33 * 8)) - return -EIO; - - fregs = get_fpu_regs(child); - - for (i = 0; i < 32; i++) - __get_user(fregs[i], i + (__u64 __user *) data); - - __get_user(child->thread.fpu.fcr31, data + 64); - - /* FIR may not be written. */ + /* System call number may have been changed */ + mips_syscall_update_nr(child, regs); return 0; } @@ -174,9 +128,9 @@ int ptrace_get_watch_regs(struct task_struct *child, enum pt_watch_style style; int i; - if (!cpu_has_watch || current_cpu_data.watch_reg_use_cnt == 0) + if (!cpu_has_watch || boot_cpu_data.watch_reg_use_cnt == 0) return -EIO; - if (!access_ok(VERIFY_WRITE, addr, sizeof(struct pt_watch_regs))) + if (!access_ok(addr, sizeof(struct pt_watch_regs))) return -EIO; #ifdef CONFIG_32BIT @@ -188,14 +142,15 @@ int ptrace_get_watch_regs(struct task_struct *child, #endif __put_user(style, &addr->style); - __put_user(current_cpu_data.watch_reg_use_cnt, + __put_user(boot_cpu_data.watch_reg_use_cnt, &addr->WATCH_STYLE.num_valid); - for (i = 0; i < current_cpu_data.watch_reg_use_cnt; i++) { + for (i = 0; i < boot_cpu_data.watch_reg_use_cnt; i++) { __put_user(child->thread.watch.mips3264.watchlo[i], &addr->WATCH_STYLE.watchlo[i]); - __put_user(child->thread.watch.mips3264.watchhi[i] & 0xfff, + __put_user(child->thread.watch.mips3264.watchhi[i] & + (MIPS_WATCHHI_MASK | MIPS_WATCHHI_IRW), &addr->WATCH_STYLE.watchhi[i]); - __put_user(current_cpu_data.watch_reg_masks[i], + __put_user(boot_cpu_data.watch_reg_masks[i], &addr->WATCH_STYLE.watch_masks[i]); } for (; i < 8; i++) { @@ -215,12 +170,12 @@ int ptrace_set_watch_regs(struct task_struct *child, unsigned long lt[NUM_WATCH_REGS]; u16 ht[NUM_WATCH_REGS]; - if (!cpu_has_watch || current_cpu_data.watch_reg_use_cnt == 0) + if (!cpu_has_watch || boot_cpu_data.watch_reg_use_cnt == 0) return -EIO; - if (!access_ok(VERIFY_READ, addr, sizeof(struct pt_watch_regs))) + if (!access_ok(addr, sizeof(struct pt_watch_regs))) return -EIO; /* Check the values. */ - for (i = 0; i < current_cpu_data.watch_reg_use_cnt; i++) { + for (i = 0; i < boot_cpu_data.watch_reg_use_cnt; i++) { __get_user(lt[i], &addr->WATCH_STYLE.watchlo[i]); #ifdef CONFIG_32BIT if (lt[i] & __UA_LIMIT) @@ -235,12 +190,12 @@ int ptrace_set_watch_regs(struct task_struct *child, } #endif __get_user(ht[i], &addr->WATCH_STYLE.watchhi[i]); - if (ht[i] & ~0xff8) + if (ht[i] & ~MIPS_WATCHHI_MASK) return -EINVAL; } /* Install them. */ - for (i = 0; i < current_cpu_data.watch_reg_use_cnt; i++) { - if (lt[i] & 7) + for (i = 0; i < boot_cpu_data.watch_reg_use_cnt; i++) { + if (lt[i] & MIPS_WATCHLO_IRW) watch_active = 1; child->thread.watch.mips3264.watchlo[i] = lt[i]; /* Set the G bit. */ @@ -255,6 +210,877 @@ int ptrace_set_watch_regs(struct task_struct *child, return 0; } +/* regset get/set implementations */ + +#if defined(CONFIG_32BIT) || defined(CONFIG_MIPS32_O32) + +static int gpr32_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + struct pt_regs *regs = task_pt_regs(target); + u32 uregs[ELF_NGREG] = {}; + + mips_dump_regs32(uregs, regs); + return membuf_write(&to, uregs, sizeof(uregs)); +} + +static int gpr32_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + struct pt_regs *regs = task_pt_regs(target); + u32 uregs[ELF_NGREG]; + unsigned start, num_regs, i; + int err; + + start = pos / sizeof(u32); + num_regs = count / sizeof(u32); + + if (start + num_regs > ELF_NGREG) + return -EIO; + + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, uregs, 0, + sizeof(uregs)); + if (err) + return err; + + for (i = start; i < num_regs; i++) { + /* + * Cast all values to signed here so that if this is a 64-bit + * kernel, the supplied 32-bit values will be sign extended. + */ + switch (i) { + case MIPS32_EF_R1 ... MIPS32_EF_R25: + /* k0/k1 are ignored. */ + case MIPS32_EF_R28 ... MIPS32_EF_R31: + regs->regs[i - MIPS32_EF_R0] = (s32)uregs[i]; + break; + case MIPS32_EF_LO: + regs->lo = (s32)uregs[i]; + break; + case MIPS32_EF_HI: + regs->hi = (s32)uregs[i]; + break; + case MIPS32_EF_CP0_EPC: + regs->cp0_epc = (s32)uregs[i]; + break; + } + } + + /* System call number may have been changed */ + mips_syscall_update_nr(target, regs); + + return 0; +} + +#endif /* CONFIG_32BIT || CONFIG_MIPS32_O32 */ + +#ifdef CONFIG_64BIT + +static int gpr64_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + struct pt_regs *regs = task_pt_regs(target); + u64 uregs[ELF_NGREG] = {}; + + mips_dump_regs64(uregs, regs); + return membuf_write(&to, uregs, sizeof(uregs)); +} + +static int gpr64_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + struct pt_regs *regs = task_pt_regs(target); + u64 uregs[ELF_NGREG]; + unsigned start, num_regs, i; + int err; + + start = pos / sizeof(u64); + num_regs = count / sizeof(u64); + + if (start + num_regs > ELF_NGREG) + return -EIO; + + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, uregs, 0, + sizeof(uregs)); + if (err) + return err; + + for (i = start; i < num_regs; i++) { + switch (i) { + case MIPS64_EF_R1 ... MIPS64_EF_R25: + /* k0/k1 are ignored. */ + case MIPS64_EF_R28 ... MIPS64_EF_R31: + regs->regs[i - MIPS64_EF_R0] = uregs[i]; + break; + case MIPS64_EF_LO: + regs->lo = uregs[i]; + break; + case MIPS64_EF_HI: + regs->hi = uregs[i]; + break; + case MIPS64_EF_CP0_EPC: + regs->cp0_epc = uregs[i]; + break; + } + } + + /* System call number may have been changed */ + mips_syscall_update_nr(target, regs); + + return 0; +} + +#endif /* CONFIG_64BIT */ + + +#ifdef CONFIG_MIPS_FP_SUPPORT + +/* + * Poke at FCSR according to its mask. Set the Cause bits even + * if a corresponding Enable bit is set. This will be noticed at + * the time the thread is switched to and SIGFPE thrown accordingly. + */ +static void ptrace_setfcr31(struct task_struct *child, u32 value) +{ + u32 fcr31; + u32 mask; + + fcr31 = child->thread.fpu.fcr31; + mask = boot_cpu_data.fpu_msk31; + child->thread.fpu.fcr31 = (value & ~mask) | (fcr31 & mask); +} + +int ptrace_getfpregs(struct task_struct *child, __u32 __user *data) +{ + int i; + + if (!access_ok(data, 33 * 8)) + return -EIO; + + if (tsk_used_math(child)) { + union fpureg *fregs = get_fpu_regs(child); + for (i = 0; i < 32; i++) + __put_user(get_fpr64(&fregs[i], 0), + i + (__u64 __user *)data); + } else { + for (i = 0; i < 32; i++) + __put_user((__u64) -1, i + (__u64 __user *) data); + } + + __put_user(child->thread.fpu.fcr31, data + 64); + __put_user(boot_cpu_data.fpu_id, data + 65); + + return 0; +} + +int ptrace_setfpregs(struct task_struct *child, __u32 __user *data) +{ + union fpureg *fregs; + u64 fpr_val; + u32 value; + int i; + + if (!access_ok(data, 33 * 8)) + return -EIO; + + init_fp_ctx(child); + fregs = get_fpu_regs(child); + + for (i = 0; i < 32; i++) { + __get_user(fpr_val, i + (__u64 __user *)data); + set_fpr64(&fregs[i], 0, fpr_val); + } + + __get_user(value, data + 64); + ptrace_setfcr31(child, value); + + /* FIR may not be written. */ + + return 0; +} + +/* + * Copy the floating-point context to the supplied NT_PRFPREG buffer, + * !CONFIG_CPU_HAS_MSA variant. FP context's general register slots + * correspond 1:1 to buffer slots. Only general registers are copied. + */ +static void fpr_get_fpa(struct task_struct *target, + struct membuf *to) +{ + membuf_write(to, &target->thread.fpu, + NUM_FPU_REGS * sizeof(elf_fpreg_t)); +} + +/* + * Copy the floating-point context to the supplied NT_PRFPREG buffer, + * CONFIG_CPU_HAS_MSA variant. Only lower 64 bits of FP context's + * general register slots are copied to buffer slots. Only general + * registers are copied. + */ +static void fpr_get_msa(struct task_struct *target, struct membuf *to) +{ + unsigned int i; + + BUILD_BUG_ON(sizeof(u64) != sizeof(elf_fpreg_t)); + for (i = 0; i < NUM_FPU_REGS; i++) + membuf_store(to, get_fpr64(&target->thread.fpu.fpr[i], 0)); +} + +/* + * Copy the floating-point context to the supplied NT_PRFPREG buffer. + * Choose the appropriate helper for general registers, and then copy + * the FCSR and FIR registers separately. + */ +static int fpr_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + if (sizeof(target->thread.fpu.fpr[0]) == sizeof(elf_fpreg_t)) + fpr_get_fpa(target, &to); + else + fpr_get_msa(target, &to); + + membuf_write(&to, &target->thread.fpu.fcr31, sizeof(u32)); + membuf_write(&to, &boot_cpu_data.fpu_id, sizeof(u32)); + return 0; +} + +/* + * Copy the supplied NT_PRFPREG buffer to the floating-point context, + * !CONFIG_CPU_HAS_MSA variant. Buffer slots correspond 1:1 to FP + * context's general register slots. Only general registers are copied. + */ +static int fpr_set_fpa(struct task_struct *target, + unsigned int *pos, unsigned int *count, + const void **kbuf, const void __user **ubuf) +{ + return user_regset_copyin(pos, count, kbuf, ubuf, + &target->thread.fpu, + 0, NUM_FPU_REGS * sizeof(elf_fpreg_t)); +} + +/* + * Copy the supplied NT_PRFPREG buffer to the floating-point context, + * CONFIG_CPU_HAS_MSA variant. Buffer slots are copied to lower 64 + * bits only of FP context's general register slots. Only general + * registers are copied. + */ +static int fpr_set_msa(struct task_struct *target, + unsigned int *pos, unsigned int *count, + const void **kbuf, const void __user **ubuf) +{ + unsigned int i; + u64 fpr_val; + int err; + + BUILD_BUG_ON(sizeof(fpr_val) != sizeof(elf_fpreg_t)); + for (i = 0; i < NUM_FPU_REGS && *count > 0; i++) { + err = user_regset_copyin(pos, count, kbuf, ubuf, + &fpr_val, i * sizeof(elf_fpreg_t), + (i + 1) * sizeof(elf_fpreg_t)); + if (err) + return err; + set_fpr64(&target->thread.fpu.fpr[i], 0, fpr_val); + } + + return 0; +} + +/* + * Copy the supplied NT_PRFPREG buffer to the floating-point context. + * Choose the appropriate helper for general registers, and then copy + * the FCSR register separately. Ignore the incoming FIR register + * contents though, as the register is read-only. + * + * We optimize for the case where `count % sizeof(elf_fpreg_t) == 0', + * which is supposed to have been guaranteed by the kernel before + * calling us, e.g. in `ptrace_regset'. We enforce that requirement, + * so that we can safely avoid preinitializing temporaries for + * partial register writes. + */ +static int fpr_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + const int fcr31_pos = NUM_FPU_REGS * sizeof(elf_fpreg_t); + const int fir_pos = fcr31_pos + sizeof(u32); + u32 fcr31; + int err; + + BUG_ON(count % sizeof(elf_fpreg_t)); + + if (pos + count > sizeof(elf_fpregset_t)) + return -EIO; + + init_fp_ctx(target); + + if (sizeof(target->thread.fpu.fpr[0]) == sizeof(elf_fpreg_t)) + err = fpr_set_fpa(target, &pos, &count, &kbuf, &ubuf); + else + err = fpr_set_msa(target, &pos, &count, &kbuf, &ubuf); + if (err) + return err; + + if (count > 0) { + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &fcr31, + fcr31_pos, fcr31_pos + sizeof(u32)); + if (err) + return err; + + ptrace_setfcr31(target, fcr31); + } + + if (count > 0) { + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + fir_pos, fir_pos + sizeof(u32)); + return 0; + } + + return err; +} + +/* Copy the FP mode setting to the supplied NT_MIPS_FP_MODE buffer. */ +static int fp_mode_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + return membuf_store(&to, (int)mips_get_process_fp_mode(target)); +} + +/* + * Copy the supplied NT_MIPS_FP_MODE buffer to the FP mode setting. + * + * We optimize for the case where `count % sizeof(int) == 0', which + * is supposed to have been guaranteed by the kernel before calling + * us, e.g. in `ptrace_regset'. We enforce that requirement, so + * that we can safely avoid preinitializing temporaries for partial + * mode writes. + */ +static int fp_mode_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int fp_mode; + int err; + + BUG_ON(count % sizeof(int)); + + if (pos + count > sizeof(fp_mode)) + return -EIO; + + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &fp_mode, 0, + sizeof(fp_mode)); + if (err) + return err; + + if (count > 0) + err = mips_set_process_fp_mode(target, fp_mode); + + return err; +} + +#endif /* CONFIG_MIPS_FP_SUPPORT */ + +#ifdef CONFIG_CPU_HAS_MSA + +struct msa_control_regs { + unsigned int fir; + unsigned int fcsr; + unsigned int msair; + unsigned int msacsr; +}; + +static void copy_pad_fprs(struct task_struct *target, + const struct user_regset *regset, + struct membuf *to, + unsigned int live_sz) +{ + int i, j; + unsigned long long fill = ~0ull; + unsigned int cp_sz, pad_sz; + + cp_sz = min(regset->size, live_sz); + pad_sz = regset->size - cp_sz; + WARN_ON(pad_sz % sizeof(fill)); + + for (i = 0; i < NUM_FPU_REGS; i++) { + membuf_write(to, &target->thread.fpu.fpr[i], cp_sz); + for (j = 0; j < (pad_sz / sizeof(fill)); j++) + membuf_store(to, fill); + } +} + +static int msa_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + const unsigned int wr_size = NUM_FPU_REGS * regset->size; + const struct msa_control_regs ctrl_regs = { + .fir = boot_cpu_data.fpu_id, + .fcsr = target->thread.fpu.fcr31, + .msair = boot_cpu_data.msa_id, + .msacsr = target->thread.fpu.msacsr, + }; + + if (!tsk_used_math(target)) { + /* The task hasn't used FP or MSA, fill with 0xff */ + copy_pad_fprs(target, regset, &to, 0); + } else if (!test_tsk_thread_flag(target, TIF_MSA_CTX_LIVE)) { + /* Copy scalar FP context, fill the rest with 0xff */ + copy_pad_fprs(target, regset, &to, 8); + } else if (sizeof(target->thread.fpu.fpr[0]) == regset->size) { + /* Trivially copy the vector registers */ + membuf_write(&to, &target->thread.fpu.fpr, wr_size); + } else { + /* Copy as much context as possible, fill the rest with 0xff */ + copy_pad_fprs(target, regset, &to, + sizeof(target->thread.fpu.fpr[0])); + } + + return membuf_write(&to, &ctrl_regs, sizeof(ctrl_regs)); +} + +static int msa_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + const unsigned int wr_size = NUM_FPU_REGS * regset->size; + struct msa_control_regs ctrl_regs; + unsigned int cp_sz; + int i, err, start; + + init_fp_ctx(target); + + if (sizeof(target->thread.fpu.fpr[0]) == regset->size) { + /* Trivially copy the vector registers */ + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fpu.fpr, + 0, wr_size); + } else { + /* Copy as much context as possible */ + cp_sz = min_t(unsigned int, regset->size, + sizeof(target->thread.fpu.fpr[0])); + + i = start = err = 0; + for (; i < NUM_FPU_REGS; i++, start += regset->size) { + err |= user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fpu.fpr[i], + start, start + cp_sz); + } + } + + if (!err) + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl_regs, + wr_size, wr_size + sizeof(ctrl_regs)); + if (!err) { + target->thread.fpu.fcr31 = ctrl_regs.fcsr & ~FPU_CSR_ALL_X; + target->thread.fpu.msacsr = ctrl_regs.msacsr & ~MSA_CSR_CAUSEF; + } + + return err; +} + +#endif /* CONFIG_CPU_HAS_MSA */ + +#if defined(CONFIG_32BIT) || defined(CONFIG_MIPS32_O32) + +/* + * Copy the DSP context to the supplied 32-bit NT_MIPS_DSP buffer. + */ +static int dsp32_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + u32 dspregs[NUM_DSP_REGS + 1]; + unsigned int i; + + BUG_ON(to.left % sizeof(u32)); + + if (!cpu_has_dsp) + return -EIO; + + for (i = 0; i < NUM_DSP_REGS; i++) + dspregs[i] = target->thread.dsp.dspr[i]; + dspregs[NUM_DSP_REGS] = target->thread.dsp.dspcontrol; + return membuf_write(&to, dspregs, sizeof(dspregs)); +} + +/* + * Copy the supplied 32-bit NT_MIPS_DSP buffer to the DSP context. + */ +static int dsp32_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + unsigned int start, num_regs, i; + u32 dspregs[NUM_DSP_REGS + 1]; + int err; + + BUG_ON(count % sizeof(u32)); + + if (!cpu_has_dsp) + return -EIO; + + start = pos / sizeof(u32); + num_regs = count / sizeof(u32); + + if (start + num_regs > NUM_DSP_REGS + 1) + return -EIO; + + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, dspregs, 0, + sizeof(dspregs)); + if (err) + return err; + + for (i = start; i < num_regs; i++) + switch (i) { + case 0 ... NUM_DSP_REGS - 1: + target->thread.dsp.dspr[i] = (s32)dspregs[i]; + break; + case NUM_DSP_REGS: + target->thread.dsp.dspcontrol = (s32)dspregs[i]; + break; + } + + return 0; +} + +#endif /* CONFIG_32BIT || CONFIG_MIPS32_O32 */ + +#ifdef CONFIG_64BIT + +/* + * Copy the DSP context to the supplied 64-bit NT_MIPS_DSP buffer. + */ +static int dsp64_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + u64 dspregs[NUM_DSP_REGS + 1]; + unsigned int i; + + BUG_ON(to.left % sizeof(u64)); + + if (!cpu_has_dsp) + return -EIO; + + for (i = 0; i < NUM_DSP_REGS; i++) + dspregs[i] = target->thread.dsp.dspr[i]; + dspregs[NUM_DSP_REGS] = target->thread.dsp.dspcontrol; + return membuf_write(&to, dspregs, sizeof(dspregs)); +} + +/* + * Copy the supplied 64-bit NT_MIPS_DSP buffer to the DSP context. + */ +static int dsp64_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + unsigned int start, num_regs, i; + u64 dspregs[NUM_DSP_REGS + 1]; + int err; + + BUG_ON(count % sizeof(u64)); + + if (!cpu_has_dsp) + return -EIO; + + start = pos / sizeof(u64); + num_regs = count / sizeof(u64); + + if (start + num_regs > NUM_DSP_REGS + 1) + return -EIO; + + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, dspregs, 0, + sizeof(dspregs)); + if (err) + return err; + + for (i = start; i < num_regs; i++) + switch (i) { + case 0 ... NUM_DSP_REGS - 1: + target->thread.dsp.dspr[i] = dspregs[i]; + break; + case NUM_DSP_REGS: + target->thread.dsp.dspcontrol = dspregs[i]; + break; + } + + return 0; +} + +#endif /* CONFIG_64BIT */ + +/* + * Determine whether the DSP context is present. + */ +static int dsp_active(struct task_struct *target, + const struct user_regset *regset) +{ + return cpu_has_dsp ? NUM_DSP_REGS + 1 : -ENODEV; +} + +enum mips_regset { + REGSET_GPR, + REGSET_DSP, +#ifdef CONFIG_MIPS_FP_SUPPORT + REGSET_FPR, + REGSET_FP_MODE, +#endif +#ifdef CONFIG_CPU_HAS_MSA + REGSET_MSA, +#endif +}; + +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define REG_OFFSET_NAME(reg, r) { \ + .name = #reg, \ + .offset = offsetof(struct pt_regs, r) \ +} + +#define REG_OFFSET_END { \ + .name = NULL, \ + .offset = 0 \ +} + +static const struct pt_regs_offset regoffset_table[] = { + REG_OFFSET_NAME(r0, regs[0]), + REG_OFFSET_NAME(r1, regs[1]), + REG_OFFSET_NAME(r2, regs[2]), + REG_OFFSET_NAME(r3, regs[3]), + REG_OFFSET_NAME(r4, regs[4]), + REG_OFFSET_NAME(r5, regs[5]), + REG_OFFSET_NAME(r6, regs[6]), + REG_OFFSET_NAME(r7, regs[7]), + REG_OFFSET_NAME(r8, regs[8]), + REG_OFFSET_NAME(r9, regs[9]), + REG_OFFSET_NAME(r10, regs[10]), + REG_OFFSET_NAME(r11, regs[11]), + REG_OFFSET_NAME(r12, regs[12]), + REG_OFFSET_NAME(r13, regs[13]), + REG_OFFSET_NAME(r14, regs[14]), + REG_OFFSET_NAME(r15, regs[15]), + REG_OFFSET_NAME(r16, regs[16]), + REG_OFFSET_NAME(r17, regs[17]), + REG_OFFSET_NAME(r18, regs[18]), + REG_OFFSET_NAME(r19, regs[19]), + REG_OFFSET_NAME(r20, regs[20]), + REG_OFFSET_NAME(r21, regs[21]), + REG_OFFSET_NAME(r22, regs[22]), + REG_OFFSET_NAME(r23, regs[23]), + REG_OFFSET_NAME(r24, regs[24]), + REG_OFFSET_NAME(r25, regs[25]), + REG_OFFSET_NAME(r26, regs[26]), + REG_OFFSET_NAME(r27, regs[27]), + REG_OFFSET_NAME(r28, regs[28]), + REG_OFFSET_NAME(r29, regs[29]), + REG_OFFSET_NAME(r30, regs[30]), + REG_OFFSET_NAME(r31, regs[31]), + REG_OFFSET_NAME(c0_status, cp0_status), + REG_OFFSET_NAME(hi, hi), + REG_OFFSET_NAME(lo, lo), +#ifdef CONFIG_CPU_HAS_SMARTMIPS + REG_OFFSET_NAME(acx, acx), +#endif + REG_OFFSET_NAME(c0_badvaddr, cp0_badvaddr), + REG_OFFSET_NAME(c0_cause, cp0_cause), + REG_OFFSET_NAME(c0_epc, cp0_epc), +#ifdef CONFIG_CPU_CAVIUM_OCTEON + REG_OFFSET_NAME(mpl0, mpl[0]), + REG_OFFSET_NAME(mpl1, mpl[1]), + REG_OFFSET_NAME(mpl2, mpl[2]), + REG_OFFSET_NAME(mtp0, mtp[0]), + REG_OFFSET_NAME(mtp1, mtp[1]), + REG_OFFSET_NAME(mtp2, mtp[2]), +#endif + REG_OFFSET_END, +}; + +/** + * regs_query_register_offset() - query register offset from its name + * @name: the name of a register + * + * regs_query_register_offset() returns the offset of a register in struct + * pt_regs from its name. If the name is invalid, this returns -EINVAL; + */ +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + + return -EINVAL; +} + +#if defined(CONFIG_32BIT) || defined(CONFIG_MIPS32_O32) + +static const struct user_regset mips_regsets[] = { + [REGSET_GPR] = { + USER_REGSET_NOTE_TYPE(PRSTATUS), + .n = ELF_NGREG, + .size = sizeof(unsigned int), + .align = sizeof(unsigned int), + .regset_get = gpr32_get, + .set = gpr32_set, + }, + [REGSET_DSP] = { + USER_REGSET_NOTE_TYPE(MIPS_DSP), + .n = NUM_DSP_REGS + 1, + .size = sizeof(u32), + .align = sizeof(u32), + .regset_get = dsp32_get, + .set = dsp32_set, + .active = dsp_active, + }, +#ifdef CONFIG_MIPS_FP_SUPPORT + [REGSET_FPR] = { + USER_REGSET_NOTE_TYPE(PRFPREG), + .n = ELF_NFPREG, + .size = sizeof(elf_fpreg_t), + .align = sizeof(elf_fpreg_t), + .regset_get = fpr_get, + .set = fpr_set, + }, + [REGSET_FP_MODE] = { + USER_REGSET_NOTE_TYPE(MIPS_FP_MODE), + .n = 1, + .size = sizeof(int), + .align = sizeof(int), + .regset_get = fp_mode_get, + .set = fp_mode_set, + }, +#endif +#ifdef CONFIG_CPU_HAS_MSA + [REGSET_MSA] = { + USER_REGSET_NOTE_TYPE(MIPS_MSA), + .n = NUM_FPU_REGS + 1, + .size = 16, + .align = 16, + .regset_get = msa_get, + .set = msa_set, + }, +#endif +}; + +static const struct user_regset_view user_mips_view = { + .name = "mips", + .e_machine = ELF_ARCH, + .ei_osabi = ELF_OSABI, + .regsets = mips_regsets, + .n = ARRAY_SIZE(mips_regsets), +}; + +#endif /* CONFIG_32BIT || CONFIG_MIPS32_O32 */ + +#ifdef CONFIG_64BIT + +static const struct user_regset mips64_regsets[] = { + [REGSET_GPR] = { + USER_REGSET_NOTE_TYPE(PRSTATUS), + .n = ELF_NGREG, + .size = sizeof(unsigned long), + .align = sizeof(unsigned long), + .regset_get = gpr64_get, + .set = gpr64_set, + }, + [REGSET_DSP] = { + USER_REGSET_NOTE_TYPE(MIPS_DSP), + .n = NUM_DSP_REGS + 1, + .size = sizeof(u64), + .align = sizeof(u64), + .regset_get = dsp64_get, + .set = dsp64_set, + .active = dsp_active, + }, +#ifdef CONFIG_MIPS_FP_SUPPORT + [REGSET_FP_MODE] = { + USER_REGSET_NOTE_TYPE(MIPS_FP_MODE), + .n = 1, + .size = sizeof(int), + .align = sizeof(int), + .regset_get = fp_mode_get, + .set = fp_mode_set, + }, + [REGSET_FPR] = { + USER_REGSET_NOTE_TYPE(PRFPREG), + .n = ELF_NFPREG, + .size = sizeof(elf_fpreg_t), + .align = sizeof(elf_fpreg_t), + .regset_get = fpr_get, + .set = fpr_set, + }, +#endif +#ifdef CONFIG_CPU_HAS_MSA + [REGSET_MSA] = { + USER_REGSET_NOTE_TYPE(MIPS_MSA), + .n = NUM_FPU_REGS + 1, + .size = 16, + .align = 16, + .regset_get = msa_get, + .set = msa_set, + }, +#endif +}; + +static const struct user_regset_view user_mips64_view = { + .name = "mips64", + .e_machine = ELF_ARCH, + .ei_osabi = ELF_OSABI, + .regsets = mips64_regsets, + .n = ARRAY_SIZE(mips64_regsets), +}; + +#ifdef CONFIG_MIPS32_N32 + +static const struct user_regset_view user_mipsn32_view = { + .name = "mipsn32", + .e_flags = EF_MIPS_ABI2, + .e_machine = ELF_ARCH, + .ei_osabi = ELF_OSABI, + .regsets = mips64_regsets, + .n = ARRAY_SIZE(mips64_regsets), +}; + +#endif /* CONFIG_MIPS32_N32 */ + +#endif /* CONFIG_64BIT */ + +const struct user_regset_view *task_user_regset_view(struct task_struct *task) +{ +#ifdef CONFIG_32BIT + return &user_mips_view; +#else +#ifdef CONFIG_MIPS32_O32 + if (test_tsk_thread_flag(task, TIF_32BIT_REGS)) + return &user_mips_view; +#endif +#ifdef CONFIG_MIPS32_N32 + if (test_tsk_thread_flag(task, TIF_32BIT_ADDR)) + return &user_mipsn32_view; +#endif + return &user_mips64_view; +#endif +} + long arch_ptrace(struct task_struct *child, long request, unsigned long addr, unsigned long data) { @@ -282,28 +1108,40 @@ long arch_ptrace(struct task_struct *child, long request, case 0 ... 31: tmp = regs->regs[addr]; break; - case FPR_BASE ... FPR_BASE + 31: - if (tsk_used_math(child)) { - fpureg_t *fregs = get_fpu_regs(child); +#ifdef CONFIG_MIPS_FP_SUPPORT + case FPR_BASE ... FPR_BASE + 31: { + union fpureg *fregs; + + if (!tsk_used_math(child)) { + /* FP not yet used */ + tmp = -1; + break; + } + fregs = get_fpu_regs(child); #ifdef CONFIG_32BIT + if (test_tsk_thread_flag(child, TIF_32BIT_FPREGS)) { /* * The odd registers are actually the high * order bits of the values stored in the even - * registers - unless we're using r2k_switch.S. + * registers. */ - if (addr & 1) - tmp = (unsigned long) (fregs[((addr & ~1) - 32)] >> 32); - else - tmp = (unsigned long) (fregs[(addr - 32)] & 0xffffffff); -#endif -#ifdef CONFIG_64BIT - tmp = fregs[addr - FPR_BASE]; -#endif - } else { - tmp = -1; /* FP not yet used */ + tmp = get_fpr32(&fregs[(addr & ~1) - FPR_BASE], + addr & 1); + break; } +#endif + tmp = get_fpr64(&fregs[addr - FPR_BASE], 0); + break; + } + case FPC_CSR: + tmp = child->thread.fpu.fcr31; break; + case FPC_EIR: + /* implementation / version register */ + tmp = boot_cpu_data.fpu_id; + break; +#endif case PC: tmp = regs->cp0_epc; break; @@ -324,47 +1162,6 @@ long arch_ptrace(struct task_struct *child, long request, tmp = regs->acx; break; #endif - case FPC_CSR: - tmp = child->thread.fpu.fcr31; - break; - case FPC_EIR: { /* implementation / version register */ - unsigned int flags; -#ifdef CONFIG_MIPS_MT_SMTC - unsigned long irqflags; - unsigned int mtflags; -#endif /* CONFIG_MIPS_MT_SMTC */ - - preempt_disable(); - if (!cpu_has_fpu) { - preempt_enable(); - break; - } - -#ifdef CONFIG_MIPS_MT_SMTC - /* Read-modify-write of Status must be atomic */ - local_irq_save(irqflags); - mtflags = dmt(); -#endif /* CONFIG_MIPS_MT_SMTC */ - if (cpu_has_mipsmt) { - unsigned int vpflags = dvpe(); - flags = read_c0_status(); - __enable_fpu(); - __asm__ __volatile__("cfc1\t%0,$0": "=r" (tmp)); - write_c0_status(flags); - evpe(vpflags); - } else { - flags = read_c0_status(); - __enable_fpu(); - __asm__ __volatile__("cfc1\t%0,$0": "=r" (tmp)); - write_c0_status(flags); - } -#ifdef CONFIG_MIPS_MT_SMTC - emt(mtflags); - local_irq_restore(irqflags); -#endif /* CONFIG_MIPS_MT_SMTC */ - preempt_enable(); - break; - } case DSP_BASE ... DSP_BASE + 5: { dspreg_t *dregs; @@ -374,7 +1171,7 @@ long arch_ptrace(struct task_struct *child, long request, goto out; } dregs = __get_dsp_regs(child); - tmp = (unsigned long) (dregs[addr - DSP_BASE]); + tmp = dregs[addr - DSP_BASE]; break; } case DSP_CONTROL: @@ -408,35 +1205,38 @@ long arch_ptrace(struct task_struct *child, long request, switch (addr) { case 0 ... 31: regs->regs[addr] = data; + /* System call number may have been changed */ + if (addr == 2) + mips_syscall_update_nr(child, regs); + else if (addr == 4 && + mips_syscall_is_indirect(child, regs)) + mips_syscall_update_nr(child, regs); break; +#ifdef CONFIG_MIPS_FP_SUPPORT case FPR_BASE ... FPR_BASE + 31: { - fpureg_t *fregs = get_fpu_regs(child); + union fpureg *fregs = get_fpu_regs(child); - if (!tsk_used_math(child)) { - /* FP not yet used */ - memset(&child->thread.fpu, ~0, - sizeof(child->thread.fpu)); - child->thread.fpu.fcr31 = 0; - } + init_fp_ctx(child); #ifdef CONFIG_32BIT - /* - * The odd registers are actually the high order bits - * of the values stored in the even registers - unless - * we're using r2k_switch.S. - */ - if (addr & 1) { - fregs[(addr & ~1) - FPR_BASE] &= 0xffffffff; - fregs[(addr & ~1) - FPR_BASE] |= ((unsigned long long) data) << 32; - } else { - fregs[addr - FPR_BASE] &= ~0xffffffffLL; - fregs[addr - FPR_BASE] |= data; + if (test_tsk_thread_flag(child, TIF_32BIT_FPREGS)) { + /* + * The odd registers are actually the high + * order bits of the values stored in the even + * registers. + */ + set_fpr32(&fregs[(addr & ~1) - FPR_BASE], + addr & 1, data); + break; } #endif -#ifdef CONFIG_64BIT - fregs[addr - FPR_BASE] = data; -#endif + set_fpr64(&fregs[addr - FPR_BASE], 0, data); break; } + case FPC_CSR: + init_fp_ctx(child); + ptrace_setfcr31(child, data); + break; +#endif case PC: regs->cp0_epc = data; break; @@ -451,9 +1251,6 @@ long arch_ptrace(struct task_struct *child, long request, regs->acx = data; break; #endif - case FPC_CSR: - child->thread.fpu.fcr31 = data; - break; case DSP_BASE ... DSP_BASE + 5: { dspreg_t *dregs; @@ -489,6 +1286,7 @@ long arch_ptrace(struct task_struct *child, long request, ret = ptrace_setregs(child, datavp); break; +#ifdef CONFIG_MIPS_FP_SUPPORT case PTRACE_GETFPREGS: ret = ptrace_getfpregs(child, datavp); break; @@ -496,7 +1294,7 @@ long arch_ptrace(struct task_struct *child, long request, case PTRACE_SETFPREGS: ret = ptrace_setfpregs(child, datavp); break; - +#endif case PTRACE_GET_THREAD_AREA: ret = put_user(task_thread_info(child)->tp_value, datalp); break; @@ -517,54 +1315,36 @@ long arch_ptrace(struct task_struct *child, long request, return ret; } -static inline int audit_arch(void) -{ - int arch = EM_MIPS; -#ifdef CONFIG_64BIT - arch |= __AUDIT_ARCH_64BIT; -#endif -#if defined(__LITTLE_ENDIAN) - arch |= __AUDIT_ARCH_LE; -#endif - return arch; -} - /* * Notification of system call entry/exit * - triggered by current->work.syscall_trace */ -asmlinkage void syscall_trace_enter(struct pt_regs *regs) +asmlinkage long syscall_trace_enter(struct pt_regs *regs) { user_exit(); - /* do the secure computing check first */ - secure_computing_strict(regs->regs[2]); + if (test_thread_flag(TIF_SYSCALL_TRACE)) { + if (ptrace_report_syscall_entry(regs)) + return -1; + } - if (!(current->ptrace & PT_PTRACED)) - goto out; + if (secure_computing()) + return -1; - if (!test_thread_flag(TIF_SYSCALL_TRACE)) - goto out; + if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) + trace_sys_enter(regs, regs->regs[2]); - /* The 0x80 provides a way for the tracing parent to distinguish - between a syscall stop and SIGTRAP delivery */ - ptrace_notify(SIGTRAP | ((current->ptrace & PT_TRACESYSGOOD) ? - 0x80 : 0)); + audit_syscall_entry(current_thread_info()->syscall, + regs->regs[4], regs->regs[5], + regs->regs[6], regs->regs[7]); /* - * this isn't the same as continuing with a signal, but it will do - * for normal use. strace only continues with a signal if the - * stopping signal is not SIGTRAP. -brl + * Negative syscall numbers are mistaken for rejected syscalls, but + * won't have had the return value set appropriately, so we do so now. */ - if (current->exit_code) { - send_sig(current->exit_code, current, 1); - current->exit_code = 0; - } - -out: - audit_syscall_entry(audit_arch(), regs->regs[2], - regs->regs[4], regs->regs[5], - regs->regs[6], regs->regs[7]); + if (current_thread_info()->syscall < 0) + syscall_set_return_value(current, regs, -ENOSYS, 0); + return current_thread_info()->syscall; } /* @@ -573,7 +1353,7 @@ out: */ asmlinkage void syscall_trace_leave(struct pt_regs *regs) { - /* + /* * We may come here right after calling schedule_user() * or do_notify_resume(), in which case we can be in RCU * user mode. @@ -582,26 +1362,11 @@ asmlinkage void syscall_trace_leave(struct pt_regs *regs) audit_syscall_exit(regs); - if (!(current->ptrace & PT_PTRACED)) - return; + if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) + trace_sys_exit(regs, regs_return_value(regs)); - if (!test_thread_flag(TIF_SYSCALL_TRACE)) - return; - - /* The 0x80 provides a way for the tracing parent to distinguish - between a syscall stop and SIGTRAP delivery */ - ptrace_notify(SIGTRAP | ((current->ptrace & PT_TRACESYSGOOD) ? - 0x80 : 0)); - - /* - * this isn't the same as continuing with a signal, but it will do - * for normal use. strace only continues with a signal if the - * stopping signal is not SIGTRAP. -brl - */ - if (current->exit_code) { - send_sig(current->exit_code, current, 1); - current->exit_code = 0; - } + if (test_thread_flag(TIF_SYSCALL_TRACE)) + ptrace_report_syscall_exit(regs, 0); user_enter(); } diff --git a/arch/mips/kernel/ptrace32.c b/arch/mips/kernel/ptrace32.c index 9486055ba660..afcf27a877cb 100644 --- a/arch/mips/kernel/ptrace32.c +++ b/arch/mips/kernel/ptrace32.c @@ -18,11 +18,11 @@ #include <linux/compat.h> #include <linux/kernel.h> #include <linux/sched.h> +#include <linux/sched/task_stack.h> #include <linux/mm.h> #include <linux/errno.h> #include <linux/ptrace.h> #include <linux/smp.h> -#include <linux/user.h> #include <linux/security.h> #include <asm/cpu.h> @@ -30,9 +30,10 @@ #include <asm/fpu.h> #include <asm/mipsregs.h> #include <asm/mipsmtregs.h> -#include <asm/pgtable.h> #include <asm/page.h> -#include <asm/uaccess.h> +#include <asm/reg.h> +#include <asm/syscall.h> +#include <linux/uaccess.h> #include <asm/bootinfo.h> /* @@ -69,8 +70,8 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, if (get_user(addrOthers, (u32 __user * __user *) (unsigned long) addr) != 0) break; - copied = access_process_vm(child, (u64)addrOthers, &tmp, - sizeof(tmp), 0); + copied = ptrace_access_vm(child, (u64)addrOthers, &tmp, + sizeof(tmp), FOLL_FORCE); if (copied != sizeof(tmp)) break; ret = put_user(tmp, (u32 __user *) (unsigned long) data); @@ -89,23 +90,37 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, case 0 ... 31: tmp = regs->regs[addr]; break; - case FPR_BASE ... FPR_BASE + 31: - if (tsk_used_math(child)) { - fpureg_t *fregs = get_fpu_regs(child); +#ifdef CONFIG_MIPS_FP_SUPPORT + case FPR_BASE ... FPR_BASE + 31: { + union fpureg *fregs; + if (!tsk_used_math(child)) { + /* FP not yet used */ + tmp = -1; + break; + } + fregs = get_fpu_regs(child); + if (test_tsk_thread_flag(child, TIF_32BIT_FPREGS)) { /* * The odd registers are actually the high * order bits of the values stored in the even - * registers - unless we're using r2k_switch.S. + * registers. */ - if (addr & 1) - tmp = (unsigned long) (fregs[((addr & ~1) - 32)] >> 32); - else - tmp = (unsigned long) (fregs[(addr - 32)] & 0xffffffff); - } else { - tmp = -1; /* FP not yet used */ + tmp = get_fpr32(&fregs[(addr & ~1) - FPR_BASE], + addr & 1); + break; } + tmp = get_fpr64(&fregs[addr - FPR_BASE], 0); + break; + } + case FPC_CSR: + tmp = child->thread.fpu.fcr31; break; + case FPC_EIR: + /* implementation / version register */ + tmp = boot_cpu_data.fpu_id; + break; +#endif /* CONFIG_MIPS_FP_SUPPORT */ case PC: tmp = regs->cp0_epc; break; @@ -121,49 +136,6 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, case MMLO: tmp = regs->lo; break; - case FPC_CSR: - tmp = child->thread.fpu.fcr31; - break; - case FPC_EIR: { /* implementation / version register */ - unsigned int flags; -#ifdef CONFIG_MIPS_MT_SMTC - unsigned int irqflags; - unsigned int mtflags; -#endif /* CONFIG_MIPS_MT_SMTC */ - - preempt_disable(); - if (!cpu_has_fpu) { - preempt_enable(); - tmp = 0; - break; - } - -#ifdef CONFIG_MIPS_MT_SMTC - /* Read-modify-write of Status must be atomic */ - local_irq_save(irqflags); - mtflags = dmt(); -#endif /* CONFIG_MIPS_MT_SMTC */ - - if (cpu_has_mipsmt) { - unsigned int vpflags = dvpe(); - flags = read_c0_status(); - __enable_fpu(); - __asm__ __volatile__("cfc1\t%0,$0": "=r" (tmp)); - write_c0_status(flags); - evpe(vpflags); - } else { - flags = read_c0_status(); - __enable_fpu(); - __asm__ __volatile__("cfc1\t%0,$0": "=r" (tmp)); - write_c0_status(flags); - } -#ifdef CONFIG_MIPS_MT_SMTC - emt(mtflags); - local_irq_restore(irqflags); -#endif /* CONFIG_MIPS_MT_SMTC */ - preempt_enable(); - break; - } case DSP_BASE ... DSP_BASE + 5: { dspreg_t *dregs; @@ -173,7 +145,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, goto out; } dregs = __get_dsp_regs(child); - tmp = (unsigned long) (dregs[addr - DSP_BASE]); + tmp = dregs[addr - DSP_BASE]; break; } case DSP_CONTROL: @@ -211,8 +183,9 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, if (get_user(addrOthers, (u32 __user * __user *) (unsigned long) addr) != 0) break; ret = 0; - if (access_process_vm(child, (u64)addrOthers, &data, - sizeof(data), 1) == sizeof(data)) + if (ptrace_access_vm(child, (u64)addrOthers, &data, + sizeof(data), + FOLL_FORCE | FOLL_WRITE) == sizeof(data)) break; ret = -EIO; break; @@ -226,9 +199,16 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, switch (addr) { case 0 ... 31: regs->regs[addr] = data; + /* System call number may have been changed */ + if (addr == 2) + mips_syscall_update_nr(child, regs); + else if (addr == 4 && + mips_syscall_is_indirect(child, regs)) + mips_syscall_update_nr(child, regs); break; +#ifdef CONFIG_MIPS_FP_SUPPORT case FPR_BASE ... FPR_BASE + 31: { - fpureg_t *fregs = get_fpu_regs(child); + union fpureg *fregs = get_fpu_regs(child); if (!tsk_used_math(child)) { /* FP not yet used */ @@ -236,22 +216,23 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, sizeof(child->thread.fpu)); child->thread.fpu.fcr31 = 0; } - /* - * The odd registers are actually the high order bits - * of the values stored in the even registers - unless - * we're using r2k_switch.S. - */ - if (addr & 1) { - fregs[(addr & ~1) - FPR_BASE] &= 0xffffffff; - fregs[(addr & ~1) - FPR_BASE] |= ((unsigned long long) data) << 32; - } else { - fregs[addr - FPR_BASE] &= ~0xffffffffLL; - /* Must cast, lest sign extension fill upper - bits! */ - fregs[addr - FPR_BASE] |= (unsigned int)data; + if (test_tsk_thread_flag(child, TIF_32BIT_FPREGS)) { + /* + * The odd registers are actually the high + * order bits of the values stored in the even + * registers. + */ + set_fpr32(&fregs[(addr & ~1) - FPR_BASE], + addr & 1, data); + break; } + set_fpr64(&fregs[addr - FPR_BASE], 0, data); break; } + case FPC_CSR: + child->thread.fpu.fcr31 = data; + break; +#endif /* CONFIG_MIPS_FP_SUPPORT */ case PC: regs->cp0_epc = data; break; @@ -261,9 +242,6 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, case MMLO: regs->lo = data; break; - case FPC_CSR: - child->thread.fpu.fcr31 = data; - break; case DSP_BASE ... DSP_BASE + 5: { dspreg_t *dregs; @@ -292,13 +270,16 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, } case PTRACE_GETREGS: - ret = ptrace_getregs(child, (__s64 __user *) (__u64) data); + ret = ptrace_getregs(child, + (struct user_pt_regs __user *) (__u64) data); break; case PTRACE_SETREGS: - ret = ptrace_setregs(child, (__s64 __user *) (__u64) data); + ret = ptrace_setregs(child, + (struct user_pt_regs __user *) (__u64) data); break; +#ifdef CONFIG_MIPS_FP_SUPPORT case PTRACE_GETFPREGS: ret = ptrace_getfpregs(child, (__u32 __user *) (__u64) data); break; @@ -306,7 +287,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, case PTRACE_SETFPREGS: ret = ptrace_setfpregs(child, (__u32 __user *) (__u64) data); break; - +#endif case PTRACE_GET_THREAD_AREA: ret = put_user(task_thread_info(child)->tp_value, (unsigned int __user *) (unsigned long) data); diff --git a/arch/mips/kernel/r2300_fpu.S b/arch/mips/kernel/r2300_fpu.S index f31063dbdaeb..c000b22e3fcd 100644 --- a/arch/mips/kernel/r2300_fpu.S +++ b/arch/mips/kernel/r2300_fpu.S @@ -11,7 +11,9 @@ * Further modifications to make this work: * Copyright (c) 1998 Harald Koerfgen */ +#include <linux/export.h> #include <asm/asm.h> +#include <asm/asmmacro.h> #include <asm/errno.h> #include <asm/fpregdef.h> #include <asm/mipsregs.h> @@ -21,105 +23,107 @@ #define EX(a,b) \ 9: a,##b; \ .section __ex_table,"a"; \ - PTR 9b,bad_stack; \ + PTR_WD 9b,fault; \ + .previous + +#define EX2(a,b) \ +9: a,##b; \ + .section __ex_table,"a"; \ + PTR_WD 9b,fault; \ + PTR_WD 9b+4,fault; \ .previous - .set noreorder .set mips1 - /* Save floating point context */ + +/* + * Save a thread's fp context. + */ +LEAF(_save_fp) +EXPORT_SYMBOL(_save_fp) + fpu_save_single a0, t1 # clobbers t1 + jr ra + END(_save_fp) + +/* + * Restore a thread's fp context. + */ +LEAF(_restore_fp) + fpu_restore_single a0, t1 # clobbers t1 + jr ra + END(_restore_fp) + + .set noreorder + +/** + * _save_fp_context() - save FP context from the FPU + * @a0 - pointer to fpregs field of sigcontext + * @a1 - pointer to fpc_csr field of sigcontext + * + * Save FP context, including the 32 FP data registers and the FP + * control & status register, from the FPU to signal context. + */ LEAF(_save_fp_context) + .set push + .set hardfloat li v0, 0 # assume success - cfc1 t1,fcr31 - EX(swc1 $f0,(SC_FPREGS+0)(a0)) - EX(swc1 $f1,(SC_FPREGS+8)(a0)) - EX(swc1 $f2,(SC_FPREGS+16)(a0)) - EX(swc1 $f3,(SC_FPREGS+24)(a0)) - EX(swc1 $f4,(SC_FPREGS+32)(a0)) - EX(swc1 $f5,(SC_FPREGS+40)(a0)) - EX(swc1 $f6,(SC_FPREGS+48)(a0)) - EX(swc1 $f7,(SC_FPREGS+56)(a0)) - EX(swc1 $f8,(SC_FPREGS+64)(a0)) - EX(swc1 $f9,(SC_FPREGS+72)(a0)) - EX(swc1 $f10,(SC_FPREGS+80)(a0)) - EX(swc1 $f11,(SC_FPREGS+88)(a0)) - EX(swc1 $f12,(SC_FPREGS+96)(a0)) - EX(swc1 $f13,(SC_FPREGS+104)(a0)) - EX(swc1 $f14,(SC_FPREGS+112)(a0)) - EX(swc1 $f15,(SC_FPREGS+120)(a0)) - EX(swc1 $f16,(SC_FPREGS+128)(a0)) - EX(swc1 $f17,(SC_FPREGS+136)(a0)) - EX(swc1 $f18,(SC_FPREGS+144)(a0)) - EX(swc1 $f19,(SC_FPREGS+152)(a0)) - EX(swc1 $f20,(SC_FPREGS+160)(a0)) - EX(swc1 $f21,(SC_FPREGS+168)(a0)) - EX(swc1 $f22,(SC_FPREGS+176)(a0)) - EX(swc1 $f23,(SC_FPREGS+184)(a0)) - EX(swc1 $f24,(SC_FPREGS+192)(a0)) - EX(swc1 $f25,(SC_FPREGS+200)(a0)) - EX(swc1 $f26,(SC_FPREGS+208)(a0)) - EX(swc1 $f27,(SC_FPREGS+216)(a0)) - EX(swc1 $f28,(SC_FPREGS+224)(a0)) - EX(swc1 $f29,(SC_FPREGS+232)(a0)) - EX(swc1 $f30,(SC_FPREGS+240)(a0)) - EX(swc1 $f31,(SC_FPREGS+248)(a0)) - EX(sw t1,(SC_FPC_CSR)(a0)) - cfc1 t0,$0 # implementation/version + cfc1 t1, fcr31 + EX2(s.d $f0, 0(a0)) + EX2(s.d $f2, 16(a0)) + EX2(s.d $f4, 32(a0)) + EX2(s.d $f6, 48(a0)) + EX2(s.d $f8, 64(a0)) + EX2(s.d $f10, 80(a0)) + EX2(s.d $f12, 96(a0)) + EX2(s.d $f14, 112(a0)) + EX2(s.d $f16, 128(a0)) + EX2(s.d $f18, 144(a0)) + EX2(s.d $f20, 160(a0)) + EX2(s.d $f22, 176(a0)) + EX2(s.d $f24, 192(a0)) + EX2(s.d $f26, 208(a0)) + EX2(s.d $f28, 224(a0)) + EX2(s.d $f30, 240(a0)) jr ra - .set nomacro - EX(sw t0,(SC_FPC_EIR)(a0)) - .set macro + EX(sw t1, (a1)) + .set pop END(_save_fp_context) -/* - * Restore FPU state: - * - fp gp registers - * - cp1 status/control register +/** + * _restore_fp_context() - restore FP context to the FPU + * @a0 - pointer to fpregs field of sigcontext + * @a1 - pointer to fpc_csr field of sigcontext * - * We base the decision which registers to restore from the signal stack - * frame on the current content of c0_status, not on the content of the - * stack frame which might have been changed by the user. + * Restore FP context, including the 32 FP data registers and the FP + * control & status register, from signal context to the FPU. */ LEAF(_restore_fp_context) + .set push + .set hardfloat li v0, 0 # assume success - EX(lw t0,(SC_FPC_CSR)(a0)) - EX(lwc1 $f0,(SC_FPREGS+0)(a0)) - EX(lwc1 $f1,(SC_FPREGS+8)(a0)) - EX(lwc1 $f2,(SC_FPREGS+16)(a0)) - EX(lwc1 $f3,(SC_FPREGS+24)(a0)) - EX(lwc1 $f4,(SC_FPREGS+32)(a0)) - EX(lwc1 $f5,(SC_FPREGS+40)(a0)) - EX(lwc1 $f6,(SC_FPREGS+48)(a0)) - EX(lwc1 $f7,(SC_FPREGS+56)(a0)) - EX(lwc1 $f8,(SC_FPREGS+64)(a0)) - EX(lwc1 $f9,(SC_FPREGS+72)(a0)) - EX(lwc1 $f10,(SC_FPREGS+80)(a0)) - EX(lwc1 $f11,(SC_FPREGS+88)(a0)) - EX(lwc1 $f12,(SC_FPREGS+96)(a0)) - EX(lwc1 $f13,(SC_FPREGS+104)(a0)) - EX(lwc1 $f14,(SC_FPREGS+112)(a0)) - EX(lwc1 $f15,(SC_FPREGS+120)(a0)) - EX(lwc1 $f16,(SC_FPREGS+128)(a0)) - EX(lwc1 $f17,(SC_FPREGS+136)(a0)) - EX(lwc1 $f18,(SC_FPREGS+144)(a0)) - EX(lwc1 $f19,(SC_FPREGS+152)(a0)) - EX(lwc1 $f20,(SC_FPREGS+160)(a0)) - EX(lwc1 $f21,(SC_FPREGS+168)(a0)) - EX(lwc1 $f22,(SC_FPREGS+176)(a0)) - EX(lwc1 $f23,(SC_FPREGS+184)(a0)) - EX(lwc1 $f24,(SC_FPREGS+192)(a0)) - EX(lwc1 $f25,(SC_FPREGS+200)(a0)) - EX(lwc1 $f26,(SC_FPREGS+208)(a0)) - EX(lwc1 $f27,(SC_FPREGS+216)(a0)) - EX(lwc1 $f28,(SC_FPREGS+224)(a0)) - EX(lwc1 $f29,(SC_FPREGS+232)(a0)) - EX(lwc1 $f30,(SC_FPREGS+240)(a0)) - EX(lwc1 $f31,(SC_FPREGS+248)(a0)) + EX(lw t0, (a1)) + EX2(l.d $f0, 0(a0)) + EX2(l.d $f2, 16(a0)) + EX2(l.d $f4, 32(a0)) + EX2(l.d $f6, 48(a0)) + EX2(l.d $f8, 64(a0)) + EX2(l.d $f10, 80(a0)) + EX2(l.d $f12, 96(a0)) + EX2(l.d $f14, 112(a0)) + EX2(l.d $f16, 128(a0)) + EX2(l.d $f18, 144(a0)) + EX2(l.d $f20, 160(a0)) + EX2(l.d $f22, 176(a0)) + EX2(l.d $f24, 192(a0)) + EX2(l.d $f26, 208(a0)) + EX2(l.d $f28, 224(a0)) + EX2(l.d $f30, 240(a0)) jr ra - ctc1 t0,fcr31 + ctc1 t0, fcr31 + .set pop END(_restore_fp_context) .set reorder - .type fault@function + .type fault, @function .ent fault fault: li v0, -EFAULT jr ra diff --git a/arch/mips/kernel/r2300_switch.S b/arch/mips/kernel/r2300_switch.S index 38af83f84c4a..48e63943e6f7 100644 --- a/arch/mips/kernel/r2300_switch.S +++ b/arch/mips/kernel/r2300_switch.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * r2300_switch.S: R2300 specific task switching code. * @@ -25,24 +26,8 @@ .align 5 /* - * Offset to the current process status flags, the first 32 bytes of the - * stack are not used. - */ -#define ST_OFF (_THREAD_SIZE - 32 - PT_SIZE + PT_STATUS) - -/* - * FPU context is saved iff the process has used it's FPU in the current - * time slice as indicated by TIF_USEDFPU. In any case, the CU1 bit for user - * space STATUS register should be 0, so that a process *always* starts its - * userland with FPU disabled after each context switch. - * - * FPU will be enabled as soon as the process accesses FPU again, through - * do_cpu() trap. - */ - -/* * task_struct *resume(task_struct *prev, task_struct *next, - * struct thread_info *next_ti, int usedfpu) + * struct thread_info *next_ti) */ LEAF(resume) mfc0 t1, CP0_STATUS @@ -50,24 +35,8 @@ LEAF(resume) cpu_save_nonscratch a0 sw ra, THREAD_REG31(a0) - beqz a3, 1f - - PTR_L t3, TASK_THREAD_INFO(a0) - - /* - * clear saved user stack CU1 bit - */ - lw t0, ST_OFF(t3) - li t1, ~ST0_CU1 - and t0, t0, t1 - sw t0, ST_OFF(t3) - - fpu_save_single a0, t0 # clobbers t0 - -1: - -#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP) - PTR_L t8, __stack_chk_guard +#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) + PTR_LA t8, __stack_chk_guard LONG_L t9, TASK_STACK_CANARY(a1) LONG_S t9, 0(t8) #endif @@ -93,75 +62,3 @@ LEAF(resume) move v0, a0 jr ra END(resume) - -/* - * Save a thread's fp context. - */ -LEAF(_save_fp) - fpu_save_single a0, t1 # clobbers t1 - jr ra - END(_save_fp) - -/* - * Restore a thread's fp context. - */ -LEAF(_restore_fp) - fpu_restore_single a0, t1 # clobbers t1 - jr ra - END(_restore_fp) - -/* - * Load the FPU with signalling NANS. This bit pattern we're using has - * the property that no matter whether considered as single or as double - * precision represents signaling NANS. - * - * We initialize fcr31 to rounding to nearest, no exceptions. - */ - -#define FPU_DEFAULT 0x00000000 - -LEAF(_init_fpu) - mfc0 t0, CP0_STATUS - li t1, ST0_CU1 - or t0, t1 - mtc0 t0, CP0_STATUS - - li t1, FPU_DEFAULT - ctc1 t1, fcr31 - - li t0, -1 - - mtc1 t0, $f0 - mtc1 t0, $f1 - mtc1 t0, $f2 - mtc1 t0, $f3 - mtc1 t0, $f4 - mtc1 t0, $f5 - mtc1 t0, $f6 - mtc1 t0, $f7 - mtc1 t0, $f8 - mtc1 t0, $f9 - mtc1 t0, $f10 - mtc1 t0, $f11 - mtc1 t0, $f12 - mtc1 t0, $f13 - mtc1 t0, $f14 - mtc1 t0, $f15 - mtc1 t0, $f16 - mtc1 t0, $f17 - mtc1 t0, $f18 - mtc1 t0, $f19 - mtc1 t0, $f20 - mtc1 t0, $f21 - mtc1 t0, $f22 - mtc1 t0, $f23 - mtc1 t0, $f24 - mtc1 t0, $f25 - mtc1 t0, $f26 - mtc1 t0, $f27 - mtc1 t0, $f28 - mtc1 t0, $f29 - mtc1 t0, $f30 - mtc1 t0, $f31 - jr ra - END(_init_fpu) diff --git a/arch/mips/kernel/cpu-bugs64.c b/arch/mips/kernel/r4k-bugs64.c index 0c61df281ce6..1e300330078d 100644 --- a/arch/mips/kernel/cpu-bugs64.c +++ b/arch/mips/kernel/r4k-bugs64.c @@ -1,10 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2003, 2004, 2007 Maciej W. Rozycki - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. */ #include <linux/context_tracking.h> #include <linux/init.h> @@ -18,17 +14,19 @@ #include <asm/fpu.h> #include <asm/mipsregs.h> #include <asm/setup.h> +#include <asm/traps.h> static char bug64hit[] __initdata = "reliable operation impossible!\n%s"; static char nowar[] __initdata = - "Please report to <linux-mips@linux-mips.org>."; + "Please report to <linux-mips@vger.kernel.org>."; static char r4kwar[] __initdata = "Enable CPU_R4000_WORKAROUNDS to rectify."; static char daddiwar[] __initdata = "Enable CPU_DADDI_WORKAROUNDS to rectify."; -static inline void align_mod(const int align, const int mod) +static __always_inline __init +void align_mod(const int align, const int mod) { asm volatile( ".set push\n\t" @@ -39,11 +37,12 @@ static inline void align_mod(const int align, const int mod) ".endr\n\t" ".set pop" : - : GCC_IMM_ASM() (align), GCC_IMM_ASM() (mod)); + : "n"(align), "n"(mod)); } -static inline void mult_sh_align_mod(long *v1, long *v2, long *w, - const int align, const int mod) +static __always_inline __init +void mult_sh_align_mod(long *v1, long *v2, long *w, + const int align, const int mod) { unsigned long flags; int m1, m2; @@ -92,7 +91,7 @@ static inline void mult_sh_align_mod(long *v1, long *v2, long *w, ".set pop" : "=&r" (lv1), "=r" (lw) : "r" (m1), "r" (m2), "r" (s), "I" (0) - : "hi", "lo", GCC_REG_ACCUM); + : "hi", "lo", "$0"); /* We have to use single integers for m1 and m2 and a double * one for p to be sure the mulsidi3 gcc's RTL multiplication * instruction has the workaround applied. Older versions of @@ -117,7 +116,7 @@ static inline void mult_sh_align_mod(long *v1, long *v2, long *w, *w = lw; } -static inline void check_mult_sh(void) +static __always_inline __init void check_mult_sh(void) { long v1[8], v2[8], w[8]; int bug, fix, i; @@ -148,11 +147,11 @@ static inline void check_mult_sh(void) bug = 1; if (bug == 0) { - printk("no.\n"); + pr_cont("no.\n"); return; } - printk("yes, workaround... "); + pr_cont("yes, workaround... "); fix = 1; for (i = 0; i < 8; i++) @@ -160,15 +159,16 @@ static inline void check_mult_sh(void) fix = 0; if (fix == 1) { - printk("yes.\n"); + pr_cont("yes.\n"); return; } - printk("no.\n"); - panic(bug64hit, !R4000_WAR ? r4kwar : nowar); + pr_cont("no.\n"); + panic(bug64hit, + IS_ENABLED(CONFIG_CPU_R4000_WORKAROUNDS) ? nowar : r4kwar); } -static volatile int daddi_ov __cpuinitdata; +static volatile int daddi_ov; asmlinkage void __init do_daddi_ov(struct pt_regs *regs) { @@ -180,7 +180,7 @@ asmlinkage void __init do_daddi_ov(struct pt_regs *regs) exception_exit(prev_state); } -static inline void check_daddi(void) +static __init void check_daddi(void) { extern asmlinkage void handle_daddi_ov(void); unsigned long flags; @@ -190,7 +190,7 @@ static inline void check_daddi(void) printk("Checking for the daddi bug... "); local_irq_save(flags); - handler = set_except_vector(12, handle_daddi_ov); + handler = set_except_vector(EXCCODE_OV, handle_daddi_ov); /* * The following code fails to trigger an overflow exception * when executed on R4000 rev. 2.2 or 3.0 (PRId 00000422 or @@ -214,39 +214,40 @@ static inline void check_daddi(void) ".set pop" : "=r" (v), "=&r" (tmp) : "I" (0xffffffffffffdb9aUL), "I" (0x1234)); - set_except_vector(12, handler); + set_except_vector(EXCCODE_OV, handler); local_irq_restore(flags); if (daddi_ov) { - printk("no.\n"); + pr_cont("no.\n"); return; } - printk("yes, workaround... "); + pr_cont("yes, workaround... "); local_irq_save(flags); - handler = set_except_vector(12, handle_daddi_ov); + handler = set_except_vector(EXCCODE_OV, handle_daddi_ov); asm volatile( "addiu %1, $0, %2\n\t" "dsrl %1, %1, 1\n\t" "daddi %0, %1, %3" : "=r" (v), "=&r" (tmp) : "I" (0xffffffffffffdb9aUL), "I" (0x1234)); - set_except_vector(12, handler); + set_except_vector(EXCCODE_OV, handler); local_irq_restore(flags); if (daddi_ov) { - printk("yes.\n"); + pr_cont("yes.\n"); return; } - printk("no.\n"); - panic(bug64hit, !DADDI_WAR ? daddiwar : nowar); + pr_cont("no.\n"); + panic(bug64hit, + IS_ENABLED(CONFIG_CPU_DADDI_WORKAROUNDS) ? nowar : daddiwar); } int daddiu_bug = -1; -static inline void check_daddiu(void) +static __init void check_daddiu(void) { long v, w, tmp; @@ -288,11 +289,11 @@ static inline void check_daddiu(void) daddiu_bug = v != w; if (!daddiu_bug) { - printk("no.\n"); + pr_cont("no.\n"); return; } - printk("yes, workaround... "); + pr_cont("yes, workaround... "); asm volatile( "addiu %2, $0, %3\n\t" @@ -304,12 +305,13 @@ static inline void check_daddiu(void) : "I" (0xffffffffffffdb9aUL), "I" (0x1234)); if (v == w) { - printk("yes.\n"); + pr_cont("yes.\n"); return; } - printk("no.\n"); - panic(bug64hit, !DADDI_WAR ? daddiwar : nowar); + pr_cont("no.\n"); + panic(bug64hit, + IS_ENABLED(CONFIG_CPU_DADDI_WORKAROUNDS) ? nowar : daddiwar); } void __init check_bugs64_early(void) diff --git a/arch/mips/kernel/r4k_fpu.S b/arch/mips/kernel/r4k_fpu.S index 55ffe149dae9..4bb97ee89904 100644 --- a/arch/mips/kernel/r4k_fpu.S +++ b/arch/mips/kernel/r4k_fpu.S @@ -12,176 +12,405 @@ * Copyright (C) 2000 MIPS Technologies, Inc. * Copyright (C) 1999, 2001 Silicon Graphics, Inc. */ +#include <linux/export.h> #include <asm/asm.h> +#include <asm/asmmacro.h> #include <asm/errno.h> #include <asm/fpregdef.h> #include <asm/mipsregs.h> #include <asm/asm-offsets.h> #include <asm/regdef.h> +/* preprocessor replaces the fp in ".set fp=64" with $30 otherwise */ +#undef fp + .macro EX insn, reg, src .set push + .set hardfloat .set nomacro .ex\@: \insn \reg, \src .set pop .section __ex_table,"a" - PTR .ex\@, fault + PTR_WD .ex\@, fault .previous .endm +/* + * Save a thread's fp context. + */ +LEAF(_save_fp) +EXPORT_SYMBOL(_save_fp) +#if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPSR2) || \ + defined(CONFIG_CPU_MIPSR5) || defined(CONFIG_CPU_MIPSR6) + mfc0 t0, CP0_STATUS +#endif + fpu_save_double a0 t0 t1 # clobbers t1 + jr ra + END(_save_fp) + +/* + * Restore a thread's fp context. + */ +LEAF(_restore_fp) +#if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPSR2) || \ + defined(CONFIG_CPU_MIPSR5) || defined(CONFIG_CPU_MIPSR6) + mfc0 t0, CP0_STATUS +#endif + fpu_restore_double a0 t0 t1 # clobbers t1 + jr ra + END(_restore_fp) + +#ifdef CONFIG_CPU_HAS_MSA + +/* + * Save a thread's MSA vector context. + */ +LEAF(_save_msa) +EXPORT_SYMBOL(_save_msa) + msa_save_all a0 + jr ra + END(_save_msa) + +/* + * Restore a thread's MSA vector context. + */ +LEAF(_restore_msa) + msa_restore_all a0 + jr ra + END(_restore_msa) + +LEAF(_init_msa_upper) + msa_init_all_upper + jr ra + END(_init_msa_upper) + +#endif + .set noreorder - .set mips3 +/** + * _save_fp_context() - save FP context from the FPU + * @a0 - pointer to fpregs field of sigcontext + * @a1 - pointer to fpc_csr field of sigcontext + * + * Save FP context, including the 32 FP data registers and the FP + * control & status register, from the FPU to signal context. + */ LEAF(_save_fp_context) + .set push + .set hardfloat cfc1 t1, fcr31 + .set pop -#ifdef CONFIG_64BIT +#if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPSR2) || \ + defined(CONFIG_CPU_MIPSR5) || defined(CONFIG_CPU_MIPSR6) + .set push + .set hardfloat +#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) + .set mips32r2 + .set fp=64 + mfc0 t0, CP0_STATUS + sll t0, t0, 5 + bgez t0, 1f # skip storing odd if FR=0 + nop +#endif /* Store the 16 odd double precision registers */ - EX sdc1 $f1, SC_FPREGS+8(a0) - EX sdc1 $f3, SC_FPREGS+24(a0) - EX sdc1 $f5, SC_FPREGS+40(a0) - EX sdc1 $f7, SC_FPREGS+56(a0) - EX sdc1 $f9, SC_FPREGS+72(a0) - EX sdc1 $f11, SC_FPREGS+88(a0) - EX sdc1 $f13, SC_FPREGS+104(a0) - EX sdc1 $f15, SC_FPREGS+120(a0) - EX sdc1 $f17, SC_FPREGS+136(a0) - EX sdc1 $f19, SC_FPREGS+152(a0) - EX sdc1 $f21, SC_FPREGS+168(a0) - EX sdc1 $f23, SC_FPREGS+184(a0) - EX sdc1 $f25, SC_FPREGS+200(a0) - EX sdc1 $f27, SC_FPREGS+216(a0) - EX sdc1 $f29, SC_FPREGS+232(a0) - EX sdc1 $f31, SC_FPREGS+248(a0) + EX sdc1 $f1, 8(a0) + EX sdc1 $f3, 24(a0) + EX sdc1 $f5, 40(a0) + EX sdc1 $f7, 56(a0) + EX sdc1 $f9, 72(a0) + EX sdc1 $f11, 88(a0) + EX sdc1 $f13, 104(a0) + EX sdc1 $f15, 120(a0) + EX sdc1 $f17, 136(a0) + EX sdc1 $f19, 152(a0) + EX sdc1 $f21, 168(a0) + EX sdc1 $f23, 184(a0) + EX sdc1 $f25, 200(a0) + EX sdc1 $f27, 216(a0) + EX sdc1 $f29, 232(a0) + EX sdc1 $f31, 248(a0) +1: .set pop #endif + .set push + .set hardfloat /* Store the 16 even double precision registers */ - EX sdc1 $f0, SC_FPREGS+0(a0) - EX sdc1 $f2, SC_FPREGS+16(a0) - EX sdc1 $f4, SC_FPREGS+32(a0) - EX sdc1 $f6, SC_FPREGS+48(a0) - EX sdc1 $f8, SC_FPREGS+64(a0) - EX sdc1 $f10, SC_FPREGS+80(a0) - EX sdc1 $f12, SC_FPREGS+96(a0) - EX sdc1 $f14, SC_FPREGS+112(a0) - EX sdc1 $f16, SC_FPREGS+128(a0) - EX sdc1 $f18, SC_FPREGS+144(a0) - EX sdc1 $f20, SC_FPREGS+160(a0) - EX sdc1 $f22, SC_FPREGS+176(a0) - EX sdc1 $f24, SC_FPREGS+192(a0) - EX sdc1 $f26, SC_FPREGS+208(a0) - EX sdc1 $f28, SC_FPREGS+224(a0) - EX sdc1 $f30, SC_FPREGS+240(a0) - EX sw t1, SC_FPC_CSR(a0) + EX sdc1 $f0, 0(a0) + EX sdc1 $f2, 16(a0) + EX sdc1 $f4, 32(a0) + EX sdc1 $f6, 48(a0) + EX sdc1 $f8, 64(a0) + EX sdc1 $f10, 80(a0) + EX sdc1 $f12, 96(a0) + EX sdc1 $f14, 112(a0) + EX sdc1 $f16, 128(a0) + EX sdc1 $f18, 144(a0) + EX sdc1 $f20, 160(a0) + EX sdc1 $f22, 176(a0) + EX sdc1 $f24, 192(a0) + EX sdc1 $f26, 208(a0) + EX sdc1 $f28, 224(a0) + EX sdc1 $f30, 240(a0) + EX sw t1, 0(a1) jr ra li v0, 0 # success + .set pop END(_save_fp_context) -#ifdef CONFIG_MIPS32_COMPAT - /* Save 32-bit process floating point context */ -LEAF(_save_fp_context32) - cfc1 t1, fcr31 - - EX sdc1 $f0, SC32_FPREGS+0(a0) - EX sdc1 $f2, SC32_FPREGS+16(a0) - EX sdc1 $f4, SC32_FPREGS+32(a0) - EX sdc1 $f6, SC32_FPREGS+48(a0) - EX sdc1 $f8, SC32_FPREGS+64(a0) - EX sdc1 $f10, SC32_FPREGS+80(a0) - EX sdc1 $f12, SC32_FPREGS+96(a0) - EX sdc1 $f14, SC32_FPREGS+112(a0) - EX sdc1 $f16, SC32_FPREGS+128(a0) - EX sdc1 $f18, SC32_FPREGS+144(a0) - EX sdc1 $f20, SC32_FPREGS+160(a0) - EX sdc1 $f22, SC32_FPREGS+176(a0) - EX sdc1 $f24, SC32_FPREGS+192(a0) - EX sdc1 $f26, SC32_FPREGS+208(a0) - EX sdc1 $f28, SC32_FPREGS+224(a0) - EX sdc1 $f30, SC32_FPREGS+240(a0) - EX sw t1, SC32_FPC_CSR(a0) - cfc1 t0, $0 # implementation/version - EX sw t0, SC32_FPC_EIR(a0) - - jr ra - li v0, 0 # success - END(_save_fp_context32) -#endif - -/* - * Restore FPU state: - * - fp gp registers - * - cp1 status/control register +/** + * _restore_fp_context() - restore FP context to the FPU + * @a0 - pointer to fpregs field of sigcontext + * @a1 - pointer to fpc_csr field of sigcontext + * + * Restore FP context, including the 32 FP data registers and the FP + * control & status register, from signal context to the FPU. */ LEAF(_restore_fp_context) - EX lw t0, SC_FPC_CSR(a0) -#ifdef CONFIG_64BIT - EX ldc1 $f1, SC_FPREGS+8(a0) - EX ldc1 $f3, SC_FPREGS+24(a0) - EX ldc1 $f5, SC_FPREGS+40(a0) - EX ldc1 $f7, SC_FPREGS+56(a0) - EX ldc1 $f9, SC_FPREGS+72(a0) - EX ldc1 $f11, SC_FPREGS+88(a0) - EX ldc1 $f13, SC_FPREGS+104(a0) - EX ldc1 $f15, SC_FPREGS+120(a0) - EX ldc1 $f17, SC_FPREGS+136(a0) - EX ldc1 $f19, SC_FPREGS+152(a0) - EX ldc1 $f21, SC_FPREGS+168(a0) - EX ldc1 $f23, SC_FPREGS+184(a0) - EX ldc1 $f25, SC_FPREGS+200(a0) - EX ldc1 $f27, SC_FPREGS+216(a0) - EX ldc1 $f29, SC_FPREGS+232(a0) - EX ldc1 $f31, SC_FPREGS+248(a0) + EX lw t1, 0(a1) + +#if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPSR2) || \ + defined(CONFIG_CPU_MIPSR5) || defined(CONFIG_CPU_MIPSR6) + .set push + .set hardfloat +#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) + .set mips32r2 + .set fp=64 + mfc0 t0, CP0_STATUS + sll t0, t0, 5 + bgez t0, 1f # skip loading odd if FR=0 + nop +#endif + EX ldc1 $f1, 8(a0) + EX ldc1 $f3, 24(a0) + EX ldc1 $f5, 40(a0) + EX ldc1 $f7, 56(a0) + EX ldc1 $f9, 72(a0) + EX ldc1 $f11, 88(a0) + EX ldc1 $f13, 104(a0) + EX ldc1 $f15, 120(a0) + EX ldc1 $f17, 136(a0) + EX ldc1 $f19, 152(a0) + EX ldc1 $f21, 168(a0) + EX ldc1 $f23, 184(a0) + EX ldc1 $f25, 200(a0) + EX ldc1 $f27, 216(a0) + EX ldc1 $f29, 232(a0) + EX ldc1 $f31, 248(a0) +1: .set pop #endif - EX ldc1 $f0, SC_FPREGS+0(a0) - EX ldc1 $f2, SC_FPREGS+16(a0) - EX ldc1 $f4, SC_FPREGS+32(a0) - EX ldc1 $f6, SC_FPREGS+48(a0) - EX ldc1 $f8, SC_FPREGS+64(a0) - EX ldc1 $f10, SC_FPREGS+80(a0) - EX ldc1 $f12, SC_FPREGS+96(a0) - EX ldc1 $f14, SC_FPREGS+112(a0) - EX ldc1 $f16, SC_FPREGS+128(a0) - EX ldc1 $f18, SC_FPREGS+144(a0) - EX ldc1 $f20, SC_FPREGS+160(a0) - EX ldc1 $f22, SC_FPREGS+176(a0) - EX ldc1 $f24, SC_FPREGS+192(a0) - EX ldc1 $f26, SC_FPREGS+208(a0) - EX ldc1 $f28, SC_FPREGS+224(a0) - EX ldc1 $f30, SC_FPREGS+240(a0) - ctc1 t0, fcr31 + .set push + .set hardfloat + EX ldc1 $f0, 0(a0) + EX ldc1 $f2, 16(a0) + EX ldc1 $f4, 32(a0) + EX ldc1 $f6, 48(a0) + EX ldc1 $f8, 64(a0) + EX ldc1 $f10, 80(a0) + EX ldc1 $f12, 96(a0) + EX ldc1 $f14, 112(a0) + EX ldc1 $f16, 128(a0) + EX ldc1 $f18, 144(a0) + EX ldc1 $f20, 160(a0) + EX ldc1 $f22, 176(a0) + EX ldc1 $f24, 192(a0) + EX ldc1 $f26, 208(a0) + EX ldc1 $f28, 224(a0) + EX ldc1 $f30, 240(a0) + ctc1 t1, fcr31 + .set pop jr ra li v0, 0 # success END(_restore_fp_context) -#ifdef CONFIG_MIPS32_COMPAT -LEAF(_restore_fp_context32) - /* Restore an o32 sigcontext. */ - EX lw t0, SC32_FPC_CSR(a0) - EX ldc1 $f0, SC32_FPREGS+0(a0) - EX ldc1 $f2, SC32_FPREGS+16(a0) - EX ldc1 $f4, SC32_FPREGS+32(a0) - EX ldc1 $f6, SC32_FPREGS+48(a0) - EX ldc1 $f8, SC32_FPREGS+64(a0) - EX ldc1 $f10, SC32_FPREGS+80(a0) - EX ldc1 $f12, SC32_FPREGS+96(a0) - EX ldc1 $f14, SC32_FPREGS+112(a0) - EX ldc1 $f16, SC32_FPREGS+128(a0) - EX ldc1 $f18, SC32_FPREGS+144(a0) - EX ldc1 $f20, SC32_FPREGS+160(a0) - EX ldc1 $f22, SC32_FPREGS+176(a0) - EX ldc1 $f24, SC32_FPREGS+192(a0) - EX ldc1 $f26, SC32_FPREGS+208(a0) - EX ldc1 $f28, SC32_FPREGS+224(a0) - EX ldc1 $f30, SC32_FPREGS+240(a0) - ctc1 t0, fcr31 +#ifdef CONFIG_CPU_HAS_MSA + + .macro op_one_wr op, idx, base + .align 4 +\idx: \op \idx, 0, \base jr ra - li v0, 0 # success - END(_restore_fp_context32) + nop + .endm + + .macro op_msa_wr name, op +LEAF(\name) + .set push + .set noreorder + sll t0, a0, 4 + PTR_LA t1, 0f + PTR_ADDU t0, t0, t1 + jr t0 + nop + op_one_wr \op, 0, a1 + op_one_wr \op, 1, a1 + op_one_wr \op, 2, a1 + op_one_wr \op, 3, a1 + op_one_wr \op, 4, a1 + op_one_wr \op, 5, a1 + op_one_wr \op, 6, a1 + op_one_wr \op, 7, a1 + op_one_wr \op, 8, a1 + op_one_wr \op, 9, a1 + op_one_wr \op, 10, a1 + op_one_wr \op, 11, a1 + op_one_wr \op, 12, a1 + op_one_wr \op, 13, a1 + op_one_wr \op, 14, a1 + op_one_wr \op, 15, a1 + op_one_wr \op, 16, a1 + op_one_wr \op, 17, a1 + op_one_wr \op, 18, a1 + op_one_wr \op, 19, a1 + op_one_wr \op, 20, a1 + op_one_wr \op, 21, a1 + op_one_wr \op, 22, a1 + op_one_wr \op, 23, a1 + op_one_wr \op, 24, a1 + op_one_wr \op, 25, a1 + op_one_wr \op, 26, a1 + op_one_wr \op, 27, a1 + op_one_wr \op, 28, a1 + op_one_wr \op, 29, a1 + op_one_wr \op, 30, a1 + op_one_wr \op, 31, a1 + .set pop + END(\name) + .endm + + op_msa_wr read_msa_wr_b, st_b + op_msa_wr read_msa_wr_h, st_h + op_msa_wr read_msa_wr_w, st_w + op_msa_wr read_msa_wr_d, st_d + + op_msa_wr write_msa_wr_b, ld_b + op_msa_wr write_msa_wr_h, ld_h + op_msa_wr write_msa_wr_w, ld_w + op_msa_wr write_msa_wr_d, ld_d + +#endif /* CONFIG_CPU_HAS_MSA */ + +#ifdef CONFIG_CPU_HAS_MSA + + .macro save_msa_upper wr, off, base + .set push + .set noat +#ifdef CONFIG_64BIT + copy_s_d \wr, 1 + EX sd $1, \off(\base) +#elif defined(CONFIG_CPU_LITTLE_ENDIAN) + copy_s_w \wr, 2 + EX sw $1, \off(\base) + copy_s_w \wr, 3 + EX sw $1, (\off+4)(\base) +#else /* CONFIG_CPU_BIG_ENDIAN */ + copy_s_w \wr, 2 + EX sw $1, (\off+4)(\base) + copy_s_w \wr, 3 + EX sw $1, \off(\base) #endif + .set pop + .endm + +LEAF(_save_msa_all_upper) + save_msa_upper 0, 0x00, a0 + save_msa_upper 1, 0x08, a0 + save_msa_upper 2, 0x10, a0 + save_msa_upper 3, 0x18, a0 + save_msa_upper 4, 0x20, a0 + save_msa_upper 5, 0x28, a0 + save_msa_upper 6, 0x30, a0 + save_msa_upper 7, 0x38, a0 + save_msa_upper 8, 0x40, a0 + save_msa_upper 9, 0x48, a0 + save_msa_upper 10, 0x50, a0 + save_msa_upper 11, 0x58, a0 + save_msa_upper 12, 0x60, a0 + save_msa_upper 13, 0x68, a0 + save_msa_upper 14, 0x70, a0 + save_msa_upper 15, 0x78, a0 + save_msa_upper 16, 0x80, a0 + save_msa_upper 17, 0x88, a0 + save_msa_upper 18, 0x90, a0 + save_msa_upper 19, 0x98, a0 + save_msa_upper 20, 0xa0, a0 + save_msa_upper 21, 0xa8, a0 + save_msa_upper 22, 0xb0, a0 + save_msa_upper 23, 0xb8, a0 + save_msa_upper 24, 0xc0, a0 + save_msa_upper 25, 0xc8, a0 + save_msa_upper 26, 0xd0, a0 + save_msa_upper 27, 0xd8, a0 + save_msa_upper 28, 0xe0, a0 + save_msa_upper 29, 0xe8, a0 + save_msa_upper 30, 0xf0, a0 + save_msa_upper 31, 0xf8, a0 + jr ra + li v0, 0 + END(_save_msa_all_upper) + + .macro restore_msa_upper wr, off, base + .set push + .set noat +#ifdef CONFIG_64BIT + EX ld $1, \off(\base) + insert_d \wr, 1 +#elif defined(CONFIG_CPU_LITTLE_ENDIAN) + EX lw $1, \off(\base) + insert_w \wr, 2 + EX lw $1, (\off+4)(\base) + insert_w \wr, 3 +#else /* CONFIG_CPU_BIG_ENDIAN */ + EX lw $1, (\off+4)(\base) + insert_w \wr, 2 + EX lw $1, \off(\base) + insert_w \wr, 3 +#endif + .set pop + .endm + +LEAF(_restore_msa_all_upper) + restore_msa_upper 0, 0x00, a0 + restore_msa_upper 1, 0x08, a0 + restore_msa_upper 2, 0x10, a0 + restore_msa_upper 3, 0x18, a0 + restore_msa_upper 4, 0x20, a0 + restore_msa_upper 5, 0x28, a0 + restore_msa_upper 6, 0x30, a0 + restore_msa_upper 7, 0x38, a0 + restore_msa_upper 8, 0x40, a0 + restore_msa_upper 9, 0x48, a0 + restore_msa_upper 10, 0x50, a0 + restore_msa_upper 11, 0x58, a0 + restore_msa_upper 12, 0x60, a0 + restore_msa_upper 13, 0x68, a0 + restore_msa_upper 14, 0x70, a0 + restore_msa_upper 15, 0x78, a0 + restore_msa_upper 16, 0x80, a0 + restore_msa_upper 17, 0x88, a0 + restore_msa_upper 18, 0x90, a0 + restore_msa_upper 19, 0x98, a0 + restore_msa_upper 20, 0xa0, a0 + restore_msa_upper 21, 0xa8, a0 + restore_msa_upper 22, 0xb0, a0 + restore_msa_upper 23, 0xb8, a0 + restore_msa_upper 24, 0xc0, a0 + restore_msa_upper 25, 0xc8, a0 + restore_msa_upper 26, 0xd0, a0 + restore_msa_upper 27, 0xd8, a0 + restore_msa_upper 28, 0xe0, a0 + restore_msa_upper 29, 0xe8, a0 + restore_msa_upper 30, 0xf0, a0 + restore_msa_upper 31, 0xf8, a0 + jr ra + li v0, 0 + END(_restore_msa_all_upper) + +#endif /* CONFIG_CPU_HAS_MSA */ .set reorder - .type fault@function + .type fault, @function .ent fault fault: li v0, -EFAULT # failure jr ra diff --git a/arch/mips/kernel/r4k_switch.S b/arch/mips/kernel/r4k_switch.S index 921238a6bd26..58232ae6cfae 100644 --- a/arch/mips/kernel/r4k_switch.S +++ b/arch/mips/kernel/r4k_switch.S @@ -12,10 +12,8 @@ */ #include <asm/asm.h> #include <asm/cachectl.h> -#include <asm/fpregdef.h> #include <asm/mipsregs.h> #include <asm/asm-offsets.h> -#include <asm/pgtable-bits.h> #include <asm/regdef.h> #include <asm/stackframe.h> #include <asm/thread_info.h> @@ -23,24 +21,8 @@ #include <asm/asmmacro.h> /* - * Offset to the current process status flags, the first 32 bytes of the - * stack are not used. - */ -#define ST_OFF (_THREAD_SIZE - 32 - PT_SIZE + PT_STATUS) - -/* - * FPU context is saved iff the process has used it's FPU in the current - * time slice as indicated by _TIF_USEDFPU. In any case, the CU1 bit for user - * space STATUS register should be 0, so that a process *always* starts its - * userland with FPU disabled after each context switch. - * - * FPU will be enabled as soon as the process accesses FPU again, through - * do_cpu() trap. - */ - -/* * task_struct *resume(task_struct *prev, task_struct *next, - * struct thread_info *next_ti, int usedfpu) + * struct thread_info *next_ti) */ .align 5 LEAF(resume) @@ -49,27 +31,8 @@ cpu_save_nonscratch a0 LONG_S ra, THREAD_REG31(a0) - /* - * check if we need to save FPU registers - */ - - beqz a3, 1f - - PTR_L t3, TASK_THREAD_INFO(a0) - /* - * clear saved user stack CU1 bit - */ - LONG_L t0, ST_OFF(t3) - li t1, ~ST0_CU1 - and t0, t0, t1 - LONG_S t0, ST_OFF(t3) - - fpu_save_double a0 t0 t1 # c0_status passed in t0 - # clobbers t1 -1: - -#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP) - PTR_L t8, __stack_chk_guard +#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP) + PTR_LA t8, __stack_chk_guard LONG_L t9, TASK_STACK_CANARY(a1) LONG_S t9, 0(t8) #endif @@ -83,18 +46,6 @@ PTR_ADDU t0, $28, _THREAD_SIZE - 32 set_saved_sp t0, t1, t2 -#ifdef CONFIG_MIPS_MT_SMTC - /* Read-modify-writes of Status must be atomic on a VPE */ - mfc0 t2, CP0_TCSTATUS - ori t1, t2, TCSTATUS_IXMT - mtc0 t1, CP0_TCSTATUS - andi t2, t2, TCSTATUS_IXMT - _ehb - DMT 8 # dmt t0 - move t1,ra - jal mips_ihb - move ra,t1 -#endif /* CONFIG_MIPS_MT_SMTC */ mfc0 t1, CP0_STATUS /* Do we really need this? */ li a3, 0xff01 and t1, a3 @@ -103,149 +54,6 @@ and a2, a3 or a2, t1 mtc0 a2, CP0_STATUS -#ifdef CONFIG_MIPS_MT_SMTC - _ehb - andi t0, t0, VPECONTROL_TE - beqz t0, 1f - emt -1: - mfc0 t1, CP0_TCSTATUS - xori t1, t1, TCSTATUS_IXMT - or t1, t1, t2 - mtc0 t1, CP0_TCSTATUS - _ehb -#endif /* CONFIG_MIPS_MT_SMTC */ move v0, a0 jr ra END(resume) - -/* - * Save a thread's fp context. - */ -LEAF(_save_fp) -#ifdef CONFIG_64BIT - mfc0 t0, CP0_STATUS -#endif - fpu_save_double a0 t0 t1 # clobbers t1 - jr ra - END(_save_fp) - -/* - * Restore a thread's fp context. - */ -LEAF(_restore_fp) -#ifdef CONFIG_64BIT - mfc0 t0, CP0_STATUS -#endif - fpu_restore_double a0 t0 t1 # clobbers t1 - jr ra - END(_restore_fp) - -/* - * Load the FPU with signalling NANS. This bit pattern we're using has - * the property that no matter whether considered as single or as double - * precision represents signaling NANS. - * - * We initialize fcr31 to rounding to nearest, no exceptions. - */ - -#define FPU_DEFAULT 0x00000000 - -LEAF(_init_fpu) -#ifdef CONFIG_MIPS_MT_SMTC - /* Rather than manipulate per-VPE Status, set per-TC bit in TCStatus */ - mfc0 t0, CP0_TCSTATUS - /* Bit position is the same for Status, TCStatus */ - li t1, ST0_CU1 - or t0, t1 - mtc0 t0, CP0_TCSTATUS -#else /* Normal MIPS CU1 enable */ - mfc0 t0, CP0_STATUS - li t1, ST0_CU1 - or t0, t1 - mtc0 t0, CP0_STATUS -#endif /* CONFIG_MIPS_MT_SMTC */ - enable_fpu_hazard - - li t1, FPU_DEFAULT - ctc1 t1, fcr31 - - li t1, -1 # SNaN - -#ifdef CONFIG_64BIT - sll t0, t0, 5 - bgez t0, 1f # 16 / 32 register mode? - - dmtc1 t1, $f1 - dmtc1 t1, $f3 - dmtc1 t1, $f5 - dmtc1 t1, $f7 - dmtc1 t1, $f9 - dmtc1 t1, $f11 - dmtc1 t1, $f13 - dmtc1 t1, $f15 - dmtc1 t1, $f17 - dmtc1 t1, $f19 - dmtc1 t1, $f21 - dmtc1 t1, $f23 - dmtc1 t1, $f25 - dmtc1 t1, $f27 - dmtc1 t1, $f29 - dmtc1 t1, $f31 -1: -#endif - -#ifdef CONFIG_CPU_MIPS32 - mtc1 t1, $f0 - mtc1 t1, $f1 - mtc1 t1, $f2 - mtc1 t1, $f3 - mtc1 t1, $f4 - mtc1 t1, $f5 - mtc1 t1, $f6 - mtc1 t1, $f7 - mtc1 t1, $f8 - mtc1 t1, $f9 - mtc1 t1, $f10 - mtc1 t1, $f11 - mtc1 t1, $f12 - mtc1 t1, $f13 - mtc1 t1, $f14 - mtc1 t1, $f15 - mtc1 t1, $f16 - mtc1 t1, $f17 - mtc1 t1, $f18 - mtc1 t1, $f19 - mtc1 t1, $f20 - mtc1 t1, $f21 - mtc1 t1, $f22 - mtc1 t1, $f23 - mtc1 t1, $f24 - mtc1 t1, $f25 - mtc1 t1, $f26 - mtc1 t1, $f27 - mtc1 t1, $f28 - mtc1 t1, $f29 - mtc1 t1, $f30 - mtc1 t1, $f31 -#else - .set mips3 - dmtc1 t1, $f0 - dmtc1 t1, $f2 - dmtc1 t1, $f4 - dmtc1 t1, $f6 - dmtc1 t1, $f8 - dmtc1 t1, $f10 - dmtc1 t1, $f12 - dmtc1 t1, $f14 - dmtc1 t1, $f16 - dmtc1 t1, $f18 - dmtc1 t1, $f20 - dmtc1 t1, $f22 - dmtc1 t1, $f24 - dmtc1 t1, $f26 - dmtc1 t1, $f28 - dmtc1 t1, $f30 -#endif - jr ra - END(_init_fpu) diff --git a/arch/mips/kernel/r6000_fpu.S b/arch/mips/kernel/r6000_fpu.S deleted file mode 100644 index da0fbe46d83b..000000000000 --- a/arch/mips/kernel/r6000_fpu.S +++ /dev/null @@ -1,87 +0,0 @@ -/* - * r6000_fpu.S: Save/restore floating point context for signal handlers. - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 1996 by Ralf Baechle - * - * Multi-arch abstraction and asm macros for easier reading: - * Copyright (C) 1996 David S. Miller (davem@davemloft.net) - */ -#include <asm/asm.h> -#include <asm/fpregdef.h> -#include <asm/mipsregs.h> -#include <asm/asm-offsets.h> -#include <asm/regdef.h> - - .set noreorder - .set mips2 - /* Save floating point context */ - LEAF(_save_fp_context) - mfc0 t0,CP0_STATUS - sll t0,t0,2 - bgez t0,1f - nop - - cfc1 t1,fcr31 - /* Store the 16 double precision registers */ - sdc1 $f0,(SC_FPREGS+0)(a0) - sdc1 $f2,(SC_FPREGS+16)(a0) - sdc1 $f4,(SC_FPREGS+32)(a0) - sdc1 $f6,(SC_FPREGS+48)(a0) - sdc1 $f8,(SC_FPREGS+64)(a0) - sdc1 $f10,(SC_FPREGS+80)(a0) - sdc1 $f12,(SC_FPREGS+96)(a0) - sdc1 $f14,(SC_FPREGS+112)(a0) - sdc1 $f16,(SC_FPREGS+128)(a0) - sdc1 $f18,(SC_FPREGS+144)(a0) - sdc1 $f20,(SC_FPREGS+160)(a0) - sdc1 $f22,(SC_FPREGS+176)(a0) - sdc1 $f24,(SC_FPREGS+192)(a0) - sdc1 $f26,(SC_FPREGS+208)(a0) - sdc1 $f28,(SC_FPREGS+224)(a0) - sdc1 $f30,(SC_FPREGS+240)(a0) - jr ra - sw t0,SC_FPC_CSR(a0) -1: jr ra - nop - END(_save_fp_context) - -/* Restore FPU state: - * - fp gp registers - * - cp1 status/control register - * - * We base the decision which registers to restore from the signal stack - * frame on the current content of c0_status, not on the content of the - * stack frame which might have been changed by the user. - */ - LEAF(_restore_fp_context) - mfc0 t0,CP0_STATUS - sll t0,t0,2 - - bgez t0,1f - lw t0,SC_FPC_CSR(a0) - /* Restore the 16 double precision registers */ - ldc1 $f0,(SC_FPREGS+0)(a0) - ldc1 $f2,(SC_FPREGS+16)(a0) - ldc1 $f4,(SC_FPREGS+32)(a0) - ldc1 $f6,(SC_FPREGS+48)(a0) - ldc1 $f8,(SC_FPREGS+64)(a0) - ldc1 $f10,(SC_FPREGS+80)(a0) - ldc1 $f12,(SC_FPREGS+96)(a0) - ldc1 $f14,(SC_FPREGS+112)(a0) - ldc1 $f16,(SC_FPREGS+128)(a0) - ldc1 $f18,(SC_FPREGS+144)(a0) - ldc1 $f20,(SC_FPREGS+160)(a0) - ldc1 $f22,(SC_FPREGS+176)(a0) - ldc1 $f24,(SC_FPREGS+192)(a0) - ldc1 $f26,(SC_FPREGS+208)(a0) - ldc1 $f28,(SC_FPREGS+224)(a0) - ldc1 $f30,(SC_FPREGS+240)(a0) - jr ra - ctc1 t0,fcr31 -1: jr ra - nop - END(_restore_fp_context) diff --git a/arch/mips/kernel/relocate.c b/arch/mips/kernel/relocate.c new file mode 100644 index 000000000000..7f1c136ad850 --- /dev/null +++ b/arch/mips/kernel/relocate.c @@ -0,0 +1,466 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Support for Kernel relocation at boot time + * + * Copyright (C) 2015, Imagination Technologies Ltd. + * Authors: Matt Redfearn (matt.redfearn@mips.com) + */ +#include <asm/bootinfo.h> +#include <asm/cacheflush.h> +#include <asm/fw/fw.h> +#include <asm/sections.h> +#include <asm/setup.h> +#include <asm/timex.h> +#include <linux/elf.h> +#include <linux/kernel.h> +#include <linux/libfdt.h> +#include <linux/of_fdt.h> +#include <linux/panic_notifier.h> +#include <linux/sched/task.h> +#include <linux/start_kernel.h> +#include <linux/string.h> +#include <linux/printk.h> + +#define RELOCATED(x) ((void *)((long)x + offset)) + +extern u32 _relocation_start[]; /* End kernel image / start relocation table */ +extern u32 _relocation_end[]; /* End relocation table */ + +extern long __start___ex_table; /* Start exception table */ +extern long __stop___ex_table; /* End exception table */ + +extern void __weak plat_fdt_relocated(void *new_location); + +/* + * This function may be defined for a platform to perform any post-relocation + * fixup necessary. + * Return non-zero to abort relocation + */ +int __weak plat_post_relocation(long offset) +{ + return 0; +} + +static inline u32 __init get_synci_step(void) +{ + u32 res; + + __asm__("rdhwr %0, $1" : "=r" (res)); + + return res; +} + +static void __init sync_icache(void *kbase, unsigned long kernel_length) +{ + void *kend = kbase + kernel_length; + u32 step = get_synci_step(); + + do { + __asm__ __volatile__( + "synci 0(%0)" + : /* no output */ + : "r" (kbase)); + + kbase += step; + } while (step && kbase < kend); + + /* Completion barrier */ + __sync(); +} + +static void __init apply_r_mips_64_rel(u32 *loc_new, long offset) +{ + *(u64 *)loc_new += offset; +} + +static void __init apply_r_mips_32_rel(u32 *loc_new, long offset) +{ + *loc_new += offset; +} + +static int __init apply_r_mips_26_rel(u32 *loc_orig, u32 *loc_new, long offset) +{ + unsigned long target_addr = (*loc_orig) & 0x03ffffff; + + if (offset % 4) { + pr_err("Dangerous R_MIPS_26 REL relocation\n"); + return -ENOEXEC; + } + + /* Original target address */ + target_addr <<= 2; + target_addr += (unsigned long)loc_orig & 0xf0000000; + + /* Get the new target address */ + target_addr += offset; + + if ((target_addr & 0xf0000000) != ((unsigned long)loc_new & 0xf0000000)) { + pr_err("R_MIPS_26 REL relocation overflow\n"); + return -ENOEXEC; + } + + target_addr -= (unsigned long)loc_new & 0xf0000000; + target_addr >>= 2; + + *loc_new = (*loc_new & ~0x03ffffff) | (target_addr & 0x03ffffff); + + return 0; +} + + +static void __init apply_r_mips_hi16_rel(u32 *loc_orig, u32 *loc_new, + long offset) +{ + unsigned long insn = *loc_orig; + unsigned long target = (insn & 0xffff) << 16; /* high 16bits of target */ + + target += offset; + + *loc_new = (insn & ~0xffff) | ((target >> 16) & 0xffff); +} + +static int __init reloc_handler(u32 type, u32 *loc_orig, u32 *loc_new, + long offset) +{ + switch (type) { + case R_MIPS_64: + apply_r_mips_64_rel(loc_new, offset); + break; + case R_MIPS_32: + apply_r_mips_32_rel(loc_new, offset); + break; + case R_MIPS_26: + return apply_r_mips_26_rel(loc_orig, loc_new, offset); + case R_MIPS_HI16: + apply_r_mips_hi16_rel(loc_orig, loc_new, offset); + break; + default: + pr_err("Unhandled relocation type %d at 0x%p\n", type, + loc_orig); + return -ENOEXEC; + } + + return 0; +} + +static int __init do_relocations(void *kbase_old, void *kbase_new, long offset) +{ + u32 *r; + u32 *loc_orig; + u32 *loc_new; + int type; + int res; + + for (r = _relocation_start; r < _relocation_end; r++) { + /* Sentinel for last relocation */ + if (*r == 0) + break; + + type = (*r >> 24) & 0xff; + loc_orig = kbase_old + ((*r & 0x00ffffff) << 2); + loc_new = RELOCATED(loc_orig); + + res = reloc_handler(type, loc_orig, loc_new, offset); + if (res) + return res; + } + + return 0; +} + +/* + * The exception table is filled in by the relocs tool after vmlinux is linked. + * It must be relocated separately since there will not be any relocation + * information for it filled in by the linker. + */ +static int __init relocate_exception_table(long offset) +{ + unsigned long *etable_start, *etable_end, *e; + + etable_start = RELOCATED(&__start___ex_table); + etable_end = RELOCATED(&__stop___ex_table); + + for (e = etable_start; e < etable_end; e++) + *e += offset; + + return 0; +} + +#ifdef CONFIG_RANDOMIZE_BASE + +static inline __init unsigned long rotate_xor(unsigned long hash, + const void *area, size_t size) +{ + const typeof(hash) *ptr = PTR_ALIGN(area, sizeof(hash)); + size_t diff, i; + + diff = (void *)ptr - area; + if (unlikely(size < diff + sizeof(hash))) + return hash; + + size = ALIGN_DOWN(size - diff, sizeof(hash)); + + for (i = 0; i < size / sizeof(hash); i++) { + /* Rotate by odd number of bits and XOR. */ + hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7); + hash ^= ptr[i]; + } + + return hash; +} + +static inline __init unsigned long get_random_boot(void) +{ + unsigned long entropy = random_get_entropy(); + unsigned long hash = 0; + + /* Attempt to create a simple but unpredictable starting entropy. */ + hash = rotate_xor(hash, linux_banner, strlen(linux_banner)); + + /* Add in any runtime entropy we can get */ + hash = rotate_xor(hash, &entropy, sizeof(entropy)); + +#if defined(CONFIG_USE_OF) + /* Get any additional entropy passed in device tree */ + if (initial_boot_params) { + int node, len; + u64 *prop; + + node = fdt_path_offset(initial_boot_params, "/chosen"); + if (node >= 0) { + prop = fdt_getprop_w(initial_boot_params, node, + "kaslr-seed", &len); + if (prop && (len == sizeof(u64))) + hash = rotate_xor(hash, prop, sizeof(*prop)); + } + } +#endif /* CONFIG_USE_OF */ + + return hash; +} + +static inline __init bool kaslr_disabled(void) +{ + char *str; + +#if defined(CONFIG_CMDLINE_BOOL) + const char *builtin_cmdline = CONFIG_CMDLINE; + + str = strstr(builtin_cmdline, "nokaslr"); + if (str == builtin_cmdline || + (str > builtin_cmdline && *(str - 1) == ' ')) + return true; +#endif + str = strstr(arcs_cmdline, "nokaslr"); + if (str == arcs_cmdline || (str > arcs_cmdline && *(str - 1) == ' ')) + return true; + + return false; +} + +static inline void __init *determine_relocation_address(void) +{ + /* Choose a new address for the kernel */ + unsigned long kernel_length; + void *dest = &_text; + unsigned long offset; + + if (kaslr_disabled()) + return dest; + + kernel_length = (long)_end - (long)(&_text); + + offset = get_random_boot() << 16; + offset &= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET - 1); + if (offset < kernel_length) + offset += ALIGN(kernel_length, 0xffff); + + return RELOCATED(dest); +} + +#else + +static inline void __init *determine_relocation_address(void) +{ + /* + * Choose a new address for the kernel + * For now we'll hard code the destination + */ + return (void *)0xffffffff81000000; +} + +#endif + +static inline int __init relocation_addr_valid(void *loc_new) +{ + if ((unsigned long)loc_new & 0x0000ffff) { + /* Inappropriately aligned new location */ + return 0; + } + if ((unsigned long)loc_new < (unsigned long)&_end) { + /* New location overlaps original kernel */ + return 0; + } + return 1; +} + +static inline void __init update_kaslr_offset(unsigned long *addr, long offset) +{ + unsigned long *new_addr = (unsigned long *)RELOCATED(addr); + + *new_addr = (unsigned long)offset; +} + +#if defined(CONFIG_USE_OF) +void __weak *plat_get_fdt(void) +{ + return NULL; +} +#endif + +void *__init relocate_kernel(void) +{ + void *loc_new; + unsigned long kernel_length; + unsigned long bss_length; + long offset = 0; + int res = 1; + /* Default to original kernel entry point */ + void *kernel_entry = start_kernel; + void *fdt = NULL; + + /* Get the command line */ + fw_init_cmdline(); +#if defined(CONFIG_USE_OF) + /* Deal with the device tree */ + fdt = plat_get_fdt(); + early_init_dt_scan(fdt, __pa(fdt)); + if (boot_command_line[0]) { + /* Boot command line was passed in device tree */ + strscpy(arcs_cmdline, boot_command_line, COMMAND_LINE_SIZE); + } +#endif /* CONFIG_USE_OF */ + + kernel_length = (long)(&_relocation_start) - (long)(&_text); + bss_length = (long)&__bss_stop - (long)&__bss_start; + + loc_new = determine_relocation_address(); + + /* Sanity check relocation address */ + if (relocation_addr_valid(loc_new)) + offset = (unsigned long)loc_new - (unsigned long)(&_text); + + /* Reset the command line now so we don't end up with a duplicate */ + arcs_cmdline[0] = '\0'; + + if (offset) { + void (*fdt_relocated_)(void *) = NULL; +#if defined(CONFIG_USE_OF) + unsigned long fdt_phys = virt_to_phys(fdt); + + /* + * If built-in dtb is used then it will have been relocated + * during kernel _text relocation. If appended DTB is used + * then it will not be relocated, but it should remain + * intact in the original location. If dtb is loaded by + * the bootloader then it may need to be moved if it crosses + * the target memory area + */ + + if (fdt_phys >= virt_to_phys(RELOCATED(&_text)) && + fdt_phys <= virt_to_phys(RELOCATED(&_end))) { + void *fdt_relocated = + RELOCATED(ALIGN((long)&_end, PAGE_SIZE)); + memcpy(fdt_relocated, fdt, fdt_totalsize(fdt)); + fdt = fdt_relocated; + fdt_relocated_ = RELOCATED(&plat_fdt_relocated); + } +#endif /* CONFIG_USE_OF */ + + /* Copy the kernel to its new location */ + memcpy(loc_new, &_text, kernel_length); + + /* Perform relocations on the new kernel */ + res = do_relocations(&_text, loc_new, offset); + if (res < 0) + goto out; + + /* Sync the caches ready for execution of new kernel */ + sync_icache(loc_new, kernel_length); + + res = relocate_exception_table(offset); + if (res < 0) + goto out; + + /* + * The original .bss has already been cleared, and + * some variables such as command line parameters + * stored to it so make a copy in the new location. + */ + memcpy(RELOCATED(&__bss_start), &__bss_start, bss_length); + + /* + * If fdt was stored outside of the kernel image and + * had to be moved then update platform's state data + * with the new fdt location + */ + if (fdt_relocated_) + fdt_relocated_(fdt); + + /* + * Last chance for the platform to abort relocation. + * This may also be used by the platform to perform any + * initialisation required now that the new kernel is + * resident in memory and ready to be executed. + */ + if (plat_post_relocation(offset)) + goto out; + + /* The current thread is now within the relocated image */ + __current_thread_info = RELOCATED(&init_thread_union); + + /* Return the new kernel's entry point */ + kernel_entry = RELOCATED(start_kernel); + + /* Error may occur before, so keep it at last */ + update_kaslr_offset(&__kaslr_offset, offset); + } +out: + return kernel_entry; +} + +/* + * Show relocation information on panic. + */ +static void show_kernel_relocation(const char *level) +{ + if (__kaslr_offset > 0) { + printk(level); + pr_cont("Kernel relocated by 0x%p\n", (void *)__kaslr_offset); + pr_cont(" .text @ 0x%p\n", _text); + pr_cont(" .data @ 0x%p\n", _sdata); + pr_cont(" .bss @ 0x%p\n", __bss_start); + } +} + +static int kernel_location_notifier_fn(struct notifier_block *self, + unsigned long v, void *p) +{ + show_kernel_relocation(KERN_EMERG); + return NOTIFY_DONE; +} + +static struct notifier_block kernel_location_notifier = { + .notifier_call = kernel_location_notifier_fn +}; + +static int __init register_kernel_offset_dumper(void) +{ + atomic_notifier_chain_register(&panic_notifier_list, + &kernel_location_notifier); + return 0; +} +__initcall(register_kernel_offset_dumper); diff --git a/arch/mips/kernel/relocate_kernel.S b/arch/mips/kernel/relocate_kernel.S index 43d2d78d3287..de894a0211d7 100644 --- a/arch/mips/kernel/relocate_kernel.S +++ b/arch/mips/kernel/relocate_kernel.S @@ -1,9 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * relocate_kernel.S for kexec * Created by <nschichan@corp.free.fr> on Thu Oct 12 17:49:57 2006 - * - * This source code is licensed under the GNU General Public License, - * Version 2. See the file COPYING for more details. */ #include <asm/asm.h> @@ -13,6 +11,8 @@ #include <asm/stackframe.h> #include <asm/addrspace.h> +#include <kernel-entry-init.h> + LEAF(relocate_new_kernel) PTR_L a0, arg0 PTR_L a1, arg1 @@ -24,7 +24,13 @@ LEAF(relocate_new_kernel) process_entry: PTR_L s2, (s0) - PTR_ADD s0, s0, SZREG + PTR_ADDIU s0, s0, SZREG + + /* + * In case of a kdump/crash kernel, the indirection page is not + * populated as the kernel is directly copied to a reserved location + */ + beqz s2, done /* destination page */ and s3, s2, 0x1 @@ -55,17 +61,16 @@ copy_word: /* copy page word by word */ REG_L s5, (s2) REG_S s5, (s4) - PTR_ADD s4, s4, SZREG - PTR_ADD s2, s2, SZREG - LONG_SUB s6, s6, 1 + PTR_ADDIU s4, s4, SZREG + PTR_ADDIU s2, s2, SZREG + LONG_ADDIU s6, s6, -1 beq s6, zero, process_entry b copy_word - b process_entry done: #ifdef CONFIG_SMP /* kexec_flag reset is signal to other CPUs what kernel - was moved to it's location. Note - we need relocated address + was moved to its location. Note - we need relocated address of kexec_flag. */ bal 1f @@ -79,7 +84,7 @@ done: #ifdef CONFIG_CPU_CAVIUM_OCTEON /* We need to flush I-cache before jumping to new kernel. - * Unfortunatelly, this code is cpu-specific. + * Unfortunately, this code is cpu-specific. */ .set push .set noreorder @@ -121,11 +126,8 @@ LEAF(kexec_smp_wait) 1: LONG_L s0, (t0) bne s0, zero,1b -#ifdef CONFIG_CPU_CAVIUM_OCTEON - .set push - .set noreorder - synci 0($0) - .set pop +#ifdef USE_KEXEC_SMP_WAIT_FINAL + kexec_smp_wait_final #else sync #endif @@ -139,15 +141,14 @@ LEAF(kexec_smp_wait) #endif /* All parameters to new kernel are passed in registers a0-a3. - * kexec_args[0..3] are uses to prepare register values. + * kexec_args[0..3] are used to prepare register values. */ -kexec_args: - EXPORT(kexec_args) -arg0: PTR 0x0 -arg1: PTR 0x0 -arg2: PTR 0x0 -arg3: PTR 0x0 +EXPORT(kexec_args) +arg0: PTR_WD 0x0 +arg1: PTR_WD 0x0 +arg2: PTR_WD 0x0 +arg3: PTR_WD 0x0 .size kexec_args,PTRSIZE*4 #ifdef CONFIG_SMP @@ -156,31 +157,27 @@ arg3: PTR 0x0 * their registers a0-a3. secondary_kexec_args[0..3] are used * to prepare register values. */ -secondary_kexec_args: - EXPORT(secondary_kexec_args) -s_arg0: PTR 0x0 -s_arg1: PTR 0x0 -s_arg2: PTR 0x0 -s_arg3: PTR 0x0 +EXPORT(secondary_kexec_args) +s_arg0: PTR_WD 0x0 +s_arg1: PTR_WD 0x0 +s_arg2: PTR_WD 0x0 +s_arg3: PTR_WD 0x0 .size secondary_kexec_args,PTRSIZE*4 kexec_flag: LONG 0x1 #endif -kexec_start_address: - EXPORT(kexec_start_address) - PTR 0x0 +EXPORT(kexec_start_address) + PTR_WD 0x0 .size kexec_start_address, PTRSIZE -kexec_indirection_page: - EXPORT(kexec_indirection_page) - PTR 0 +EXPORT(kexec_indirection_page) + PTR_WD 0 .size kexec_indirection_page, PTRSIZE relocate_new_kernel_end: -relocate_new_kernel_size: - EXPORT(relocate_new_kernel_size) - PTR relocate_new_kernel_end - relocate_new_kernel +EXPORT(relocate_new_kernel_size) + PTR_WD relocate_new_kernel_end - relocate_new_kernel .size relocate_new_kernel_size, PTRSIZE diff --git a/arch/mips/kernel/reset.c b/arch/mips/kernel/reset.c index 07fc5244aed4..e7ce07b3e79b 100644 --- a/arch/mips/kernel/reset.c +++ b/arch/mips/kernel/reset.c @@ -11,7 +11,11 @@ #include <linux/pm.h> #include <linux/types.h> #include <linux/reboot.h> +#include <linux/delay.h> +#include <asm/compiler.h> +#include <asm/idle.h> +#include <asm/mipsregs.h> #include <asm/reboot.h> /* @@ -25,20 +29,96 @@ void (*pm_power_off)(void); EXPORT_SYMBOL(pm_power_off); +static void machine_hang(void) +{ + /* + * We're hanging the system so we don't want to be interrupted anymore. + * Any interrupt handlers that ran would at best be useless & at worst + * go awry because the system isn't in a functional state. + */ + local_irq_disable(); + + /* + * Mask all interrupts, giving us a better chance of remaining in the + * low power wait state. + */ + clear_c0_status(ST0_IM); + + while (true) { + if (cpu_has_mips_r) { + /* + * We know that the wait instruction is supported so + * make use of it directly, leaving interrupts + * disabled. + */ + asm volatile( + ".set push\n\t" + ".set " MIPS_ISA_ARCH_LEVEL "\n\t" + "wait\n\t" + ".set pop"); + } else if (cpu_wait) { + /* + * Try the cpu_wait() callback. This isn't ideal since + * it'll re-enable interrupts, but that ought to be + * harmless given that they're all masked. + */ + cpu_wait(); + local_irq_disable(); + } else { + /* + * We're going to burn some power running round the + * loop, but we don't really have a choice. This isn't + * a path we should expect to run for long during + * typical use anyway. + */ + } + + /* + * In most modern MIPS CPUs interrupts will cause the wait + * instruction to graduate even when disabled, and in some + * cases even when masked. In order to prevent a timer + * interrupt from continuously taking us out of the low power + * wait state, we clear any pending timer interrupt here. + */ + if (cpu_has_counter) + write_c0_compare(0); + } +} + void machine_restart(char *command) { if (_machine_restart) _machine_restart(command); + +#ifdef CONFIG_SMP + preempt_disable(); + smp_send_stop(); +#endif + do_kernel_restart(command); + mdelay(1000); + pr_emerg("Reboot failed -- System halted\n"); + machine_hang(); } void machine_halt(void) { if (_machine_halt) _machine_halt(); + +#ifdef CONFIG_SMP + preempt_disable(); + smp_send_stop(); +#endif + machine_hang(); } void machine_power_off(void) { - if (pm_power_off) - pm_power_off(); + do_kernel_power_off(); + +#ifdef CONFIG_SMP + preempt_disable(); + smp_send_stop(); +#endif + machine_hang(); } diff --git a/arch/mips/kernel/rtlx-mt.c b/arch/mips/kernel/rtlx-mt.c new file mode 100644 index 000000000000..ff7535de42ca --- /dev/null +++ b/arch/mips/kernel/rtlx-mt.c @@ -0,0 +1,147 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2005 MIPS Technologies, Inc. All rights reserved. + * Copyright (C) 2013 Imagination Technologies Ltd. + */ +#include <linux/device.h> +#include <linux/fs.h> +#include <linux/err.h> +#include <linux/wait.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/irq.h> + +#include <asm/mips_mt.h> +#include <asm/vpe.h> +#include <asm/rtlx.h> + +static int major; + +static void rtlx_dispatch(void) +{ + if (read_c0_cause() & read_c0_status() & C_SW0) + do_IRQ(MIPS_CPU_IRQ_BASE + MIPS_CPU_RTLX_IRQ); +} + +/* + * Interrupt handler may be called before rtlx_init has otherwise had + * a chance to run. + */ +static irqreturn_t rtlx_interrupt(int irq, void *dev_id) +{ + unsigned int vpeflags; + unsigned long flags; + int i; + + local_irq_save(flags); + vpeflags = dvpe(); + set_c0_status(0x100 << MIPS_CPU_RTLX_IRQ); + irq_enable_hazard(); + evpe(vpeflags); + local_irq_restore(flags); + + for (i = 0; i < RTLX_CHANNELS; i++) { + wake_up(&channel_wqs[i].lx_queue); + wake_up(&channel_wqs[i].rt_queue); + } + + return IRQ_HANDLED; +} + +static int rtlx_irq_num = MIPS_CPU_IRQ_BASE + MIPS_CPU_RTLX_IRQ; + +void _interrupt_sp(void) +{ + unsigned long flags; + + local_irq_save(flags); + dvpe(); + settc(1); + write_vpe_c0_cause(read_vpe_c0_cause() | C_SW0); + evpe(EVPE_ENABLE); + local_irq_restore(flags); +} + +int __init rtlx_module_init(void) +{ + struct device *dev; + int i, err; + + if (!cpu_has_mipsmt) { + pr_warn("VPE loader: not a MIPS MT capable processor\n"); + return -ENODEV; + } + + if (aprp_cpu_index() == 0) { + pr_warn("No TCs reserved for AP/SP, not initializing RTLX.\n" + "Pass maxtcs=<n> argument as kernel argument\n"); + + return -ENODEV; + } + + major = register_chrdev(0, RTLX_MODULE_NAME, &rtlx_fops); + if (major < 0) { + pr_err("rtlx_module_init: unable to register device\n"); + return major; + } + + /* initialise the wait queues */ + for (i = 0; i < RTLX_CHANNELS; i++) { + init_waitqueue_head(&channel_wqs[i].rt_queue); + init_waitqueue_head(&channel_wqs[i].lx_queue); + atomic_set(&channel_wqs[i].in_open, 0); + mutex_init(&channel_wqs[i].mutex); + + dev = device_create(&mt_class, NULL, MKDEV(major, i), NULL, + "%s%d", RTLX_MODULE_NAME, i); + if (IS_ERR(dev)) { + while (i--) + device_destroy(&mt_class, MKDEV(major, i)); + + err = PTR_ERR(dev); + goto out_chrdev; + } + } + + /* set up notifiers */ + rtlx_notify.start = rtlx_starting; + rtlx_notify.stop = rtlx_stopping; + vpe_notify(aprp_cpu_index(), &rtlx_notify); + + if (cpu_has_vint) { + aprp_hook = rtlx_dispatch; + } else { + pr_err("APRP RTLX init on non-vectored-interrupt processor\n"); + err = -ENODEV; + goto out_class; + } + + err = request_irq(rtlx_irq_num, rtlx_interrupt, 0, "RTLX", rtlx); + if (err) + goto out_class; + + return 0; + +out_class: + for (i = 0; i < RTLX_CHANNELS; i++) + device_destroy(&mt_class, MKDEV(major, i)); +out_chrdev: + unregister_chrdev(major, RTLX_MODULE_NAME); + + return err; +} + +void __exit rtlx_module_exit(void) +{ + int i; + + for (i = 0; i < RTLX_CHANNELS; i++) + device_destroy(&mt_class, MKDEV(major, i)); + + unregister_chrdev(major, RTLX_MODULE_NAME); + + aprp_hook = NULL; +} diff --git a/arch/mips/kernel/rtlx.c b/arch/mips/kernel/rtlx.c index d763f11e35e2..18c509c59f33 100644 --- a/arch/mips/kernel/rtlx.c +++ b/arch/mips/kernel/rtlx.c @@ -1,114 +1,53 @@ /* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * * Copyright (C) 2005 MIPS Technologies, Inc. All rights reserved. * Copyright (C) 2005, 06 Ralf Baechle (ralf@linux-mips.org) - * - * This program is free software; you can distribute it and/or modify it - * under the terms of the GNU General Public License (Version 2) as - * published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. - * + * Copyright (C) 2013 Imagination Technologies Ltd. */ - -#include <linux/device.h> #include <linux/kernel.h> #include <linux/fs.h> -#include <linux/init.h> -#include <asm/uaccess.h> -#include <linux/list.h> -#include <linux/vmalloc.h> -#include <linux/elf.h> -#include <linux/seq_file.h> #include <linux/syscalls.h> #include <linux/moduleloader.h> -#include <linux/interrupt.h> -#include <linux/poll.h> -#include <linux/sched.h> -#include <linux/wait.h> +#include <linux/atomic.h> +#include <linux/sched/signal.h> + #include <asm/mipsmtregs.h> #include <asm/mips_mt.h> -#include <asm/cacheflush.h> -#include <linux/atomic.h> -#include <asm/cpu.h> #include <asm/processor.h> -#include <asm/vpe.h> #include <asm/rtlx.h> #include <asm/setup.h> +#include <asm/vpe.h> -static struct rtlx_info *rtlx; -static int major; -static char module_name[] = "rtlx"; - -static struct chan_waitqueues { - wait_queue_head_t rt_queue; - wait_queue_head_t lx_queue; - atomic_t in_open; - struct mutex mutex; -} channel_wqs[RTLX_CHANNELS]; - -static struct vpe_notifications notify; static int sp_stopping; - -extern void *vpe_get_shared(int index); - -static void rtlx_dispatch(void) -{ - do_IRQ(MIPS_CPU_IRQ_BASE + MIPS_CPU_RTLX_IRQ); -} - - -/* Interrupt handler may be called before rtlx_init has otherwise had - a chance to run. -*/ -static irqreturn_t rtlx_interrupt(int irq, void *dev_id) -{ - unsigned int vpeflags; - unsigned long flags; - int i; - - /* Ought not to be strictly necessary for SMTC builds */ - local_irq_save(flags); - vpeflags = dvpe(); - set_c0_status(0x100 << MIPS_CPU_RTLX_IRQ); - irq_enable_hazard(); - evpe(vpeflags); - local_irq_restore(flags); - - for (i = 0; i < RTLX_CHANNELS; i++) { - wake_up(&channel_wqs[i].lx_queue); - wake_up(&channel_wqs[i].rt_queue); - } - - return IRQ_HANDLED; -} +struct rtlx_info *rtlx; +struct chan_waitqueues channel_wqs[RTLX_CHANNELS]; +struct vpe_notifications rtlx_notify; +void (*aprp_hook)(void) = NULL; +EXPORT_SYMBOL(aprp_hook); static void __used dump_rtlx(void) { int i; - printk("id 0x%lx state %d\n", rtlx->id, rtlx->state); + pr_info("id 0x%lx state %d\n", rtlx->id, rtlx->state); for (i = 0; i < RTLX_CHANNELS; i++) { struct rtlx_channel *chan = &rtlx->channel[i]; - printk(" rt_state %d lx_state %d buffer_size %d\n", - chan->rt_state, chan->lx_state, chan->buffer_size); + pr_info(" rt_state %d lx_state %d buffer_size %d\n", + chan->rt_state, chan->lx_state, chan->buffer_size); - printk(" rt_read %d rt_write %d\n", - chan->rt_read, chan->rt_write); + pr_info(" rt_read %d rt_write %d\n", + chan->rt_read, chan->rt_write); - printk(" lx_read %d lx_write %d\n", - chan->lx_read, chan->lx_write); + pr_info(" lx_read %d lx_write %d\n", + chan->lx_read, chan->lx_write); - printk(" rt_buffer <%s>\n", chan->rt_buffer); - printk(" lx_buffer <%s>\n", chan->lx_buffer); + pr_info(" rt_buffer <%s>\n", chan->rt_buffer); + pr_info(" lx_buffer <%s>\n", chan->lx_buffer); } } @@ -116,8 +55,7 @@ static void __used dump_rtlx(void) static int rtlx_init(struct rtlx_info *rtlxi) { if (rtlxi->id != RTLX_ID) { - printk(KERN_ERR "no valid RTLX id at 0x%p 0x%lx\n", - rtlxi, rtlxi->id); + pr_err("no valid RTLX id at 0x%p 0x%lx\n", rtlxi, rtlxi->id); return -ENOEXEC; } @@ -127,20 +65,20 @@ static int rtlx_init(struct rtlx_info *rtlxi) } /* notifications */ -static void starting(int vpe) +void rtlx_starting(int vpe) { int i; sp_stopping = 0; /* force a reload of rtlx */ - rtlx=NULL; + rtlx = NULL; /* wake up any sleeping rtlx_open's */ for (i = 0; i < RTLX_CHANNELS; i++) wake_up_interruptible(&channel_wqs[i].lx_queue); } -static void stopping(int vpe) +void rtlx_stopping(int vpe) { int i; @@ -158,30 +96,30 @@ int rtlx_open(int index, int can_sleep) int ret = 0; if (index >= RTLX_CHANNELS) { - printk(KERN_DEBUG "rtlx_open index out of range\n"); + pr_debug("rtlx_open index out of range\n"); return -ENOSYS; } if (atomic_inc_return(&channel_wqs[index].in_open) > 1) { - printk(KERN_DEBUG "rtlx_open channel %d already opened\n", - index); + pr_debug("rtlx_open channel %d already opened\n", index); ret = -EBUSY; goto out_fail; } if (rtlx == NULL) { - if( (p = vpe_get_shared(tclimit)) == NULL) { - if (can_sleep) { - __wait_event_interruptible(channel_wqs[index].lx_queue, - (p = vpe_get_shared(tclimit)), ret); - if (ret) + p = vpe_get_shared(aprp_cpu_index()); + if (p == NULL) { + if (can_sleep) { + ret = __wait_event_interruptible( + channel_wqs[index].lx_queue, + (p = vpe_get_shared(aprp_cpu_index()))); + if (ret) + goto out_fail; + } else { + pr_debug("No SP program loaded, and device opened with O_NONBLOCK\n"); + ret = -ENOSYS; goto out_fail; - } else { - printk(KERN_DEBUG "No SP program loaded, and device " - "opened with O_NONBLOCK\n"); - ret = -ENOSYS; - goto out_fail; - } + } } smp_rmb(); @@ -203,24 +141,24 @@ int rtlx_open(int index, int can_sleep) ret = -ERESTARTSYS; goto out_fail; } - finish_wait(&channel_wqs[index].lx_queue, &wait); + finish_wait(&channel_wqs[index].lx_queue, + &wait); } else { - pr_err(" *vpe_get_shared is NULL. " - "Has an SP program been loaded?\n"); + pr_err(" *vpe_get_shared is NULL. Has an SP program been loaded?\n"); ret = -ENOSYS; goto out_fail; } } if ((unsigned int)*p < KSEG0) { - printk(KERN_WARNING "vpe_get_shared returned an " - "invalid pointer maybe an error code %d\n", - (int)*p); + pr_warn("vpe_get_shared returned an invalid pointer maybe an error code %d\n", + (int)*p); ret = -ENOSYS; goto out_fail; } - if ((ret = rtlx_init(*p)) < 0) + ret = rtlx_init(*p); + if (ret < 0) goto out_ret; } @@ -263,11 +201,10 @@ unsigned int rtlx_read_poll(int index, int can_sleep) /* data available to read? */ if (chan->lx_read == chan->lx_write) { if (can_sleep) { - int ret = 0; - - __wait_event_interruptible(channel_wqs[index].lx_queue, + int ret = __wait_event_interruptible( + channel_wqs[index].lx_queue, (chan->lx_read != chan->lx_write) || - sp_stopping, ret); + sp_stopping); if (ret) return ret; @@ -352,7 +289,7 @@ ssize_t rtlx_write(int index, const void __user *buffer, size_t count) size_t fl; if (rtlx == NULL) - return(-ENOSYS); + return -ENOSYS; rt = &rtlx->channel[index]; @@ -361,8 +298,8 @@ ssize_t rtlx_write(int index, const void __user *buffer, size_t count) rt_read = rt->rt_read; /* total number of bytes to copy */ - count = min(count, (size_t)write_spacefree(rt_read, rt->rt_write, - rt->buffer_size)); + count = min_t(size_t, count, write_spacefree(rt_read, rt->rt_write, + rt->buffer_size)); /* first bit from write pointer to the end of the buffer, or count */ fl = min(count, (size_t) rt->buffer_size - rt->rt_write); @@ -372,9 +309,8 @@ ssize_t rtlx_write(int index, const void __user *buffer, size_t count) goto out; /* if there's any left copy to the beginning of the buffer */ - if (count - fl) { + if (count - fl) failed = copy_from_user(rt->rt_buffer, buffer + fl, count - fl); - } out: count -= failed; @@ -384,6 +320,8 @@ out: smp_wmb(); mutex_unlock(&channel_wqs[index].mutex); + _interrupt_sp(); + return count; } @@ -398,10 +336,10 @@ static int file_release(struct inode *inode, struct file *filp) return rtlx_release(iminor(inode)); } -static unsigned int file_poll(struct file *file, poll_table * wait) +static __poll_t file_poll(struct file *file, poll_table *wait) { int minor = iminor(file_inode(file)); - unsigned int mask = 0; + __poll_t mask = 0; poll_wait(file, &channel_wqs[minor].rt_queue, wait); poll_wait(file, &channel_wqs[minor].lx_queue, wait); @@ -411,43 +349,41 @@ static unsigned int file_poll(struct file *file, poll_table * wait) /* data available to read? */ if (rtlx_read_poll(minor, 0)) - mask |= POLLIN | POLLRDNORM; + mask |= EPOLLIN | EPOLLRDNORM; /* space to write */ if (rtlx_write_poll(minor)) - mask |= POLLOUT | POLLWRNORM; + mask |= EPOLLOUT | EPOLLWRNORM; return mask; } -static ssize_t file_read(struct file *file, char __user * buffer, size_t count, - loff_t * ppos) +static ssize_t file_read(struct file *file, char __user *buffer, size_t count, + loff_t *ppos) { int minor = iminor(file_inode(file)); /* data available? */ - if (!rtlx_read_poll(minor, (file->f_flags & O_NONBLOCK) ? 0 : 1)) { - return 0; // -EAGAIN makes cat whinge - } + if (!rtlx_read_poll(minor, (file->f_flags & O_NONBLOCK) ? 0 : 1)) + return 0; /* -EAGAIN makes 'cat' whine */ return rtlx_read(minor, buffer, count); } -static ssize_t file_write(struct file *file, const char __user * buffer, - size_t count, loff_t * ppos) +static ssize_t file_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) { int minor = iminor(file_inode(file)); /* any space left... */ if (!rtlx_write_poll(minor)) { - int ret = 0; + int ret; if (file->f_flags & O_NONBLOCK) return -EAGAIN; - __wait_event_interruptible(channel_wqs[minor].rt_queue, - rtlx_write_poll(minor), - ret); + ret = __wait_event_interruptible(channel_wqs[minor].rt_queue, + rtlx_write_poll(minor)); if (ret) return ret; } @@ -455,100 +391,16 @@ static ssize_t file_write(struct file *file, const char __user * buffer, return rtlx_write(minor, buffer, count); } -static const struct file_operations rtlx_fops = { +const struct file_operations rtlx_fops = { .owner = THIS_MODULE, - .open = file_open, + .open = file_open, .release = file_release, .write = file_write, - .read = file_read, - .poll = file_poll, + .read = file_read, + .poll = file_poll, .llseek = noop_llseek, }; -static struct irqaction rtlx_irq = { - .handler = rtlx_interrupt, - .name = "RTLX", -}; - -static int rtlx_irq_num = MIPS_CPU_IRQ_BASE + MIPS_CPU_RTLX_IRQ; - -static char register_chrdev_failed[] __initdata = - KERN_ERR "rtlx_module_init: unable to register device\n"; - -static int __init rtlx_module_init(void) -{ - struct device *dev; - int i, err; - - if (!cpu_has_mipsmt) { - printk("VPE loader: not a MIPS MT capable processor\n"); - return -ENODEV; - } - - if (tclimit == 0) { - printk(KERN_WARNING "No TCs reserved for AP/SP, not " - "initializing RTLX.\nPass maxtcs=<n> argument as kernel " - "argument\n"); - - return -ENODEV; - } - - major = register_chrdev(0, module_name, &rtlx_fops); - if (major < 0) { - printk(register_chrdev_failed); - return major; - } - - /* initialise the wait queues */ - for (i = 0; i < RTLX_CHANNELS; i++) { - init_waitqueue_head(&channel_wqs[i].rt_queue); - init_waitqueue_head(&channel_wqs[i].lx_queue); - atomic_set(&channel_wqs[i].in_open, 0); - mutex_init(&channel_wqs[i].mutex); - - dev = device_create(mt_class, NULL, MKDEV(major, i), NULL, - "%s%d", module_name, i); - if (IS_ERR(dev)) { - err = PTR_ERR(dev); - goto out_chrdev; - } - } - - /* set up notifiers */ - notify.start = starting; - notify.stop = stopping; - vpe_notify(tclimit, ¬ify); - - if (cpu_has_vint) - set_vi_handler(MIPS_CPU_RTLX_IRQ, rtlx_dispatch); - else { - pr_err("APRP RTLX init on non-vectored-interrupt processor\n"); - err = -ENODEV; - goto out_chrdev; - } - - rtlx_irq.dev_id = rtlx; - setup_irq(rtlx_irq_num, &rtlx_irq); - - return 0; - -out_chrdev: - for (i = 0; i < RTLX_CHANNELS; i++) - device_destroy(mt_class, MKDEV(major, i)); - - return err; -} - -static void __exit rtlx_module_exit(void) -{ - int i; - - for (i = 0; i < RTLX_CHANNELS; i++) - device_destroy(mt_class, MKDEV(major, i)); - - unregister_chrdev(major, module_name); -} - module_init(rtlx_module_init); module_exit(rtlx_module_exit); diff --git a/arch/mips/kernel/scall32-o32.S b/arch/mips/kernel/scall32-o32.S index e774bb1088b5..4947a4f39e37 100644 --- a/arch/mips/kernel/scall32-o32.S +++ b/arch/mips/kernel/scall32-o32.S @@ -6,6 +6,7 @@ * Copyright (C) 1995-99, 2000- 02, 06 Ralf Baechle <ralf@linux-mips.org> * Copyright (C) 2001 MIPS Technologies, Inc. * Copyright (C) 2004 Thiemo Seufer + * Copyright (C) 2014 Imagination Technologies Ltd. */ #include <linux/errno.h> #include <asm/asm.h> @@ -18,12 +19,8 @@ #include <asm/sysmips.h> #include <asm/thread_info.h> #include <asm/unistd.h> -#include <asm/war.h> #include <asm/asm-offsets.h> -/* Highest syscall used of any syscall flavour */ -#define MAX_SYSCALL_NO __NR_O32_Linux + __NR_O32_Linux_syscalls - .align 5 NESTED(handle_sys, PT_SIZE, sp) .set noat @@ -34,27 +31,79 @@ NESTED(handle_sys, PT_SIZE, sp) lw t1, PT_EPC(sp) # skip syscall on return - subu v0, v0, __NR_O32_Linux # check syscall number - sltiu t0, v0, __NR_O32_Linux_syscalls + 1 addiu t1, 4 # skip to next instruction sw t1, PT_EPC(sp) - beqz t0, illegal_syscall - - sll t0, v0, 3 - la t1, sys_call_table - addu t1, t0 - lw t2, (t1) # syscall routine - lw t3, 4(t1) # >= 0 if we need stack arguments - beqz t2, illegal_syscall sw a3, PT_R26(sp) # save a3 for syscall restarting - bgez t3, stackargs -stack_done: + /* + * More than four arguments. Try to deal with it by copying the + * stack arguments from the user stack to the kernel stack. + * This Sucks (TM). + */ + lw t0, PT_R29(sp) # get old user stack pointer + + /* + * We intentionally keep the kernel stack a little below the top of + * userspace so we don't have to do a slower byte accurate check here. + */ + addu t4, t0, 32 + bltz t4, bad_stack # -> sp is bad + + /* + * Ok, copy the args from the luser stack to the kernel stack. + */ + + .set push + .set noreorder + .set nomacro + +load_a4: user_lw(t5, 16(t0)) # argument #5 from usp +load_a5: user_lw(t6, 20(t0)) # argument #6 from usp +load_a6: user_lw(t7, 24(t0)) # argument #7 from usp +load_a7: user_lw(t8, 28(t0)) # argument #8 from usp +loads_done: + + sw t5, PT_ARG4(sp) # argument #5 to ksp + sw t6, PT_ARG5(sp) # argument #6 to ksp + sw t7, PT_ARG6(sp) # argument #7 to ksp + sw t8, PT_ARG7(sp) # argument #8 to ksp + .set pop + + .section __ex_table,"a" + PTR_WD load_a4, bad_stack_a4 + PTR_WD load_a5, bad_stack_a5 + PTR_WD load_a6, bad_stack_a6 + PTR_WD load_a7, bad_stack_a7 + .previous + + /* + * syscall number is in v0 unless we called syscall(__NR_###) + * where the real syscall number is in a0 + */ + subu t2, v0, __NR_O32_Linux + bnez t2, 1f /* __NR_syscall at offset 0 */ + LONG_S a0, TI_SYSCALL($28) # Save a0 as syscall number + b 2f +1: + LONG_S v0, TI_SYSCALL($28) # Save v0 as syscall number +2: + lw t0, TI_FLAGS($28) # syscall tracing enabled? li t1, _TIF_WORK_SYSCALL_ENTRY and t0, t1 bnez t0, syscall_trace_entry # -> yes +syscall_common: + subu v0, v0, __NR_O32_Linux # check syscall number + sltiu t0, v0, __NR_O32_Linux_syscalls + beqz t0, illegal_syscall + + sll t0, v0, 2 + la t1, sys_call_table + addu t1, t0 + lw t2, (t1) # syscall routine + + beqz t2, illegal_syscall jalr t2 # Do The Real Thing (TM) @@ -75,95 +124,27 @@ o32_syscall_exit: syscall_trace_entry: SAVE_STATIC - move s0, t2 move a0, sp + jal syscall_trace_enter - move t0, s0 + bltz v0, 1f # seccomp failed? Skip syscall + RESTORE_STATIC + lw v0, PT_R2(sp) # Restore syscall (maybe modified) lw a0, PT_R4(sp) # Restore argument registers lw a1, PT_R5(sp) lw a2, PT_R6(sp) lw a3, PT_R7(sp) - jalr t0 + j syscall_common - li t0, -EMAXERRNO - 1 # error? - sltu t0, t0, v0 - sw t0, PT_R7(sp) # set error flag - beqz t0, 1f - - lw t1, PT_R2(sp) # syscall number - negu v0 # error - sw t1, PT_R0(sp) # save it for syscall restarting -1: sw v0, PT_R2(sp) # result - - j syscall_exit +1: j syscall_exit /* ------------------------------------------------------------------------ */ /* - * More than four arguments. Try to deal with it by copying the - * stack arguments from the user stack to the kernel stack. - * This Sucks (TM). - */ -stackargs: - lw t0, PT_R29(sp) # get old user stack pointer - - /* - * We intentionally keep the kernel stack a little below the top of - * userspace so we don't have to do a slower byte accurate check here. - */ - lw t5, TI_ADDR_LIMIT($28) - addu t4, t0, 32 - and t5, t4 - bltz t5, bad_stack # -> sp is bad - - /* Ok, copy the args from the luser stack to the kernel stack. - * t3 is the precomputed number of instruction bytes needed to - * load or store arguments 6-8. - */ - - la t1, 5f # load up to 3 arguments - subu t1, t3 -1: lw t5, 16(t0) # argument #5 from usp - .set push - .set noreorder - .set nomacro - jr t1 - addiu t1, 6f - 5f - -2: lw t8, 28(t0) # argument #8 from usp -3: lw t7, 24(t0) # argument #7 from usp -4: lw t6, 20(t0) # argument #6 from usp -5: jr t1 - sw t5, 16(sp) # argument #5 to ksp - -#ifdef CONFIG_CPU_MICROMIPS - sw t8, 28(sp) # argument #8 to ksp - nop - sw t7, 24(sp) # argument #7 to ksp - nop - sw t6, 20(sp) # argument #6 to ksp - nop -#else - sw t8, 28(sp) # argument #8 to ksp - sw t7, 24(sp) # argument #7 to ksp - sw t6, 20(sp) # argument #6 to ksp -#endif -6: j stack_done # go back - nop - .set pop - - .section __ex_table,"a" - PTR 1b,bad_stack - PTR 2b,bad_stack - PTR 3b,bad_stack - PTR 4b,bad_stack - .previous - - /* - * The stackpointer for a call with more than 4 arguments is bad. - * We probably should handle this case a bit more drastic. + * Our open-coded access area sanity test for the stack pointer + * failed. We probably should handle this case a bit more drastic. */ bad_stack: li v0, EFAULT @@ -172,6 +153,22 @@ bad_stack: sw t0, PT_R7(sp) j o32_syscall_exit +bad_stack_a4: + li t5, 0 + b load_a5 + +bad_stack_a5: + li t6, 0 + b load_a6 + +bad_stack_a6: + li t7, 0 + b load_a7 + +bad_stack_a7: + li t8, 0 + b loads_done + /* * The system call does not exist in this kernel */ @@ -185,17 +182,12 @@ illegal_syscall: LEAF(sys_syscall) subu t0, a0, __NR_O32_Linux # check syscall number - sltiu v0, t0, __NR_O32_Linux_syscalls + 1 + sltiu v0, t0, __NR_O32_Linux_syscalls beqz t0, einval # do not recurse - sll t1, t0, 3 + sll t1, t0, 2 beqz v0, einval lw t2, sys_call_table(t1) # syscall routine - /* Some syscalls like execve get their arguments from struct pt_regs - and claim zero arguments in the syscall table. Thus we have to - assume the worst case and shuffle around all potential arguments. - If you want performance, don't use indirect syscalls. */ - move a0, a1 # shift argument registers move a1, a2 move a2, a3 @@ -206,11 +198,6 @@ illegal_syscall: sw t4, 16(sp) sw t5, 20(sp) sw t6, 24(sp) - sw a0, PT_R4(sp) # .. and push back a0 - a3, some - sw a1, PT_R5(sp) # syscalls expect them there - sw a2, PT_R6(sp) - sw a3, PT_R7(sp) - sw a3, PT_R26(sp) # update a3 for syscall restarting jr t2 /* Unreached */ @@ -218,260 +205,6 @@ einval: li v0, -ENOSYS jr ra END(sys_syscall) - .macro fifty ptr, nargs, from=1, to=50 - sys \ptr \nargs - .if \to-\from - fifty \ptr,\nargs,"(\from+1)",\to - .endif - .endm - - .macro mille ptr, nargs, from=1, to=20 - fifty \ptr,\nargs - .if \to-\from - mille \ptr,\nargs,"(\from+1)",\to - .endif - .endm - - .macro syscalltable - sys sys_syscall 8 /* 4000 */ - sys sys_exit 1 - sys __sys_fork 0 - sys sys_read 3 - sys sys_write 3 - sys sys_open 3 /* 4005 */ - sys sys_close 1 - sys sys_waitpid 3 - sys sys_creat 2 - sys sys_link 2 - sys sys_unlink 1 /* 4010 */ - sys sys_execve 0 - sys sys_chdir 1 - sys sys_time 1 - sys sys_mknod 3 - sys sys_chmod 2 /* 4015 */ - sys sys_lchown 3 - sys sys_ni_syscall 0 - sys sys_ni_syscall 0 /* was sys_stat */ - sys sys_lseek 3 - sys sys_getpid 0 /* 4020 */ - sys sys_mount 5 - sys sys_oldumount 1 - sys sys_setuid 1 - sys sys_getuid 0 - sys sys_stime 1 /* 4025 */ - sys sys_ptrace 4 - sys sys_alarm 1 - sys sys_ni_syscall 0 /* was sys_fstat */ - sys sys_pause 0 - sys sys_utime 2 /* 4030 */ - sys sys_ni_syscall 0 - sys sys_ni_syscall 0 - sys sys_access 2 - sys sys_nice 1 - sys sys_ni_syscall 0 /* 4035 */ - sys sys_sync 0 - sys sys_kill 2 - sys sys_rename 2 - sys sys_mkdir 2 - sys sys_rmdir 1 /* 4040 */ - sys sys_dup 1 - sys sysm_pipe 0 - sys sys_times 1 - sys sys_ni_syscall 0 - sys sys_brk 1 /* 4045 */ - sys sys_setgid 1 - sys sys_getgid 0 - sys sys_ni_syscall 0 /* was signal(2) */ - sys sys_geteuid 0 - sys sys_getegid 0 /* 4050 */ - sys sys_acct 1 - sys sys_umount 2 - sys sys_ni_syscall 0 - sys sys_ioctl 3 - sys sys_fcntl 3 /* 4055 */ - sys sys_ni_syscall 2 - sys sys_setpgid 2 - sys sys_ni_syscall 0 - sys sys_olduname 1 - sys sys_umask 1 /* 4060 */ - sys sys_chroot 1 - sys sys_ustat 2 - sys sys_dup2 2 - sys sys_getppid 0 - sys sys_getpgrp 0 /* 4065 */ - sys sys_setsid 0 - sys sys_sigaction 3 - sys sys_sgetmask 0 - sys sys_ssetmask 1 - sys sys_setreuid 2 /* 4070 */ - sys sys_setregid 2 - sys sys_sigsuspend 0 - sys sys_sigpending 1 - sys sys_sethostname 2 - sys sys_setrlimit 2 /* 4075 */ - sys sys_getrlimit 2 - sys sys_getrusage 2 - sys sys_gettimeofday 2 - sys sys_settimeofday 2 - sys sys_getgroups 2 /* 4080 */ - sys sys_setgroups 2 - sys sys_ni_syscall 0 /* old_select */ - sys sys_symlink 2 - sys sys_ni_syscall 0 /* was sys_lstat */ - sys sys_readlink 3 /* 4085 */ - sys sys_uselib 1 - sys sys_swapon 2 - sys sys_reboot 3 - sys sys_old_readdir 3 - sys sys_mips_mmap 6 /* 4090 */ - sys sys_munmap 2 - sys sys_truncate 2 - sys sys_ftruncate 2 - sys sys_fchmod 2 - sys sys_fchown 3 /* 4095 */ - sys sys_getpriority 2 - sys sys_setpriority 3 - sys sys_ni_syscall 0 - sys sys_statfs 2 - sys sys_fstatfs 2 /* 4100 */ - sys sys_ni_syscall 0 /* was ioperm(2) */ - sys sys_socketcall 2 - sys sys_syslog 3 - sys sys_setitimer 3 - sys sys_getitimer 2 /* 4105 */ - sys sys_newstat 2 - sys sys_newlstat 2 - sys sys_newfstat 2 - sys sys_uname 1 - sys sys_ni_syscall 0 /* 4110 was iopl(2) */ - sys sys_vhangup 0 - sys sys_ni_syscall 0 /* was sys_idle() */ - sys sys_ni_syscall 0 /* was sys_vm86 */ - sys sys_wait4 4 - sys sys_swapoff 1 /* 4115 */ - sys sys_sysinfo 1 - sys sys_ipc 6 - sys sys_fsync 1 - sys sys_sigreturn 0 - sys __sys_clone 6 /* 4120 */ - sys sys_setdomainname 2 - sys sys_newuname 1 - sys sys_ni_syscall 0 /* sys_modify_ldt */ - sys sys_adjtimex 1 - sys sys_mprotect 3 /* 4125 */ - sys sys_sigprocmask 3 - sys sys_ni_syscall 0 /* was create_module */ - sys sys_init_module 5 - sys sys_delete_module 1 - sys sys_ni_syscall 0 /* 4130 was get_kernel_syms */ - sys sys_quotactl 4 - sys sys_getpgid 1 - sys sys_fchdir 1 - sys sys_bdflush 2 - sys sys_sysfs 3 /* 4135 */ - sys sys_personality 1 - sys sys_ni_syscall 0 /* for afs_syscall */ - sys sys_setfsuid 1 - sys sys_setfsgid 1 - sys sys_llseek 5 /* 4140 */ - sys sys_getdents 3 - sys sys_select 5 - sys sys_flock 2 - sys sys_msync 3 - sys sys_readv 3 /* 4145 */ - sys sys_writev 3 - sys sys_cacheflush 3 - sys sys_cachectl 3 - sys sys_sysmips 4 - sys sys_ni_syscall 0 /* 4150 */ - sys sys_getsid 1 - sys sys_fdatasync 1 - sys sys_sysctl 1 - sys sys_mlock 2 - sys sys_munlock 2 /* 4155 */ - sys sys_mlockall 1 - sys sys_munlockall 0 - sys sys_sched_setparam 2 - sys sys_sched_getparam 2 - sys sys_sched_setscheduler 3 /* 4160 */ - sys sys_sched_getscheduler 1 - sys sys_sched_yield 0 - sys sys_sched_get_priority_max 1 - sys sys_sched_get_priority_min 1 - sys sys_sched_rr_get_interval 2 /* 4165 */ - sys sys_nanosleep, 2 - sys sys_mremap, 5 - sys sys_accept 3 - sys sys_bind 3 - sys sys_connect 3 /* 4170 */ - sys sys_getpeername 3 - sys sys_getsockname 3 - sys sys_getsockopt 5 - sys sys_listen 2 - sys sys_recv 4 /* 4175 */ - sys sys_recvfrom 6 - sys sys_recvmsg 3 - sys sys_send 4 - sys sys_sendmsg 3 - sys sys_sendto 6 /* 4180 */ - sys sys_setsockopt 5 - sys sys_shutdown 2 - sys sys_socket 3 - sys sys_socketpair 4 - sys sys_setresuid 3 /* 4185 */ - sys sys_getresuid 3 - sys sys_ni_syscall 0 /* was sys_query_module */ - sys sys_poll 3 - sys sys_ni_syscall 0 /* was nfsservctl */ - sys sys_setresgid 3 /* 4190 */ - sys sys_getresgid 3 - sys sys_prctl 5 - sys sys_rt_sigreturn 0 - sys sys_rt_sigaction 4 - sys sys_rt_sigprocmask 4 /* 4195 */ - sys sys_rt_sigpending 2 - sys sys_rt_sigtimedwait 4 - sys sys_rt_sigqueueinfo 3 - sys sys_rt_sigsuspend 0 - sys sys_pread64 6 /* 4200 */ - sys sys_pwrite64 6 - sys sys_chown 3 - sys sys_getcwd 2 - sys sys_capget 2 - sys sys_capset 2 /* 4205 */ - sys sys_sigaltstack 0 - sys sys_sendfile 4 - sys sys_ni_syscall 0 - sys sys_ni_syscall 0 - sys sys_mips_mmap2 6 /* 4210 */ - sys sys_truncate64 4 - sys sys_ftruncate64 4 - sys sys_stat64 2 - sys sys_lstat64 2 - sys sys_fstat64 2 /* 4215 */ - sys sys_pivot_root 2 - sys sys_mincore 3 - sys sys_madvise 3 - sys sys_getdents64 3 - sys sys_fcntl64 3 /* 4220 */ - sys sys_ni_syscall 0 - sys sys_gettid 0 - sys sys_readahead 5 - sys sys_setxattr 5 - sys sys_lsetxattr 5 /* 4225 */ - sys sys_fsetxattr 5 - sys sys_getxattr 4 - sys sys_lgetxattr 4 - sys sys_fgetxattr 4 - sys sys_listxattr 3 /* 4230 */ - sys sys_llistxattr 3 - sys sys_flistxattr 3 - sys sys_removexattr 2 - sys sys_lremovexattr 2 - sys sys_fremovexattr 2 /* 4235 */ - sys sys_tkill 2 - sys sys_sendfile64 5 - sys sys_futex 6 #ifdef CONFIG_MIPS_MT_FPAFF /* * For FPU affinity scheduling on MIPS MT processors, we need to @@ -480,132 +213,13 @@ einval: li v0, -ENOSYS * these hooks for the 32-bit kernel - there is no MIPS64 MT processor * atm. */ - sys mipsmt_sys_sched_setaffinity 3 - sys mipsmt_sys_sched_getaffinity 3 -#else - sys sys_sched_setaffinity 3 - sys sys_sched_getaffinity 3 /* 4240 */ +#define sys_sched_setaffinity mipsmt_sys_sched_setaffinity +#define sys_sched_getaffinity mipsmt_sys_sched_getaffinity #endif /* CONFIG_MIPS_MT_FPAFF */ - sys sys_io_setup 2 - sys sys_io_destroy 1 - sys sys_io_getevents 5 - sys sys_io_submit 3 - sys sys_io_cancel 3 /* 4245 */ - sys sys_exit_group 1 - sys sys_lookup_dcookie 4 - sys sys_epoll_create 1 - sys sys_epoll_ctl 4 - sys sys_epoll_wait 4 /* 4250 */ - sys sys_remap_file_pages 5 - sys sys_set_tid_address 1 - sys sys_restart_syscall 0 - sys sys_fadvise64_64 7 - sys sys_statfs64 3 /* 4255 */ - sys sys_fstatfs64 2 - sys sys_timer_create 3 - sys sys_timer_settime 4 - sys sys_timer_gettime 2 - sys sys_timer_getoverrun 1 /* 4260 */ - sys sys_timer_delete 1 - sys sys_clock_settime 2 - sys sys_clock_gettime 2 - sys sys_clock_getres 2 - sys sys_clock_nanosleep 4 /* 4265 */ - sys sys_tgkill 3 - sys sys_utimes 2 - sys sys_mbind 4 - sys sys_ni_syscall 0 /* sys_get_mempolicy */ - sys sys_ni_syscall 0 /* 4270 sys_set_mempolicy */ - sys sys_mq_open 4 - sys sys_mq_unlink 1 - sys sys_mq_timedsend 5 - sys sys_mq_timedreceive 5 - sys sys_mq_notify 2 /* 4275 */ - sys sys_mq_getsetattr 3 - sys sys_ni_syscall 0 /* sys_vserver */ - sys sys_waitid 5 - sys sys_ni_syscall 0 /* available, was setaltroot */ - sys sys_add_key 5 /* 4280 */ - sys sys_request_key 4 - sys sys_keyctl 5 - sys sys_set_thread_area 1 - sys sys_inotify_init 0 - sys sys_inotify_add_watch 3 /* 4285 */ - sys sys_inotify_rm_watch 2 - sys sys_migrate_pages 4 - sys sys_openat 4 - sys sys_mkdirat 3 - sys sys_mknodat 4 /* 4290 */ - sys sys_fchownat 5 - sys sys_futimesat 3 - sys sys_fstatat64 4 - sys sys_unlinkat 3 - sys sys_renameat 4 /* 4295 */ - sys sys_linkat 5 - sys sys_symlinkat 3 - sys sys_readlinkat 4 - sys sys_fchmodat 3 - sys sys_faccessat 3 /* 4300 */ - sys sys_pselect6 6 - sys sys_ppoll 5 - sys sys_unshare 1 - sys sys_splice 6 - sys sys_sync_file_range 7 /* 4305 */ - sys sys_tee 4 - sys sys_vmsplice 4 - sys sys_move_pages 6 - sys sys_set_robust_list 2 - sys sys_get_robust_list 3 /* 4310 */ - sys sys_kexec_load 4 - sys sys_getcpu 3 - sys sys_epoll_pwait 6 - sys sys_ioprio_set 3 - sys sys_ioprio_get 2 /* 4315 */ - sys sys_utimensat 4 - sys sys_signalfd 3 - sys sys_ni_syscall 0 /* was timerfd */ - sys sys_eventfd 1 - sys sys_fallocate 6 /* 4320 */ - sys sys_timerfd_create 2 - sys sys_timerfd_gettime 2 - sys sys_timerfd_settime 4 - sys sys_signalfd4 4 - sys sys_eventfd2 2 /* 4325 */ - sys sys_epoll_create1 1 - sys sys_dup3 3 - sys sys_pipe2 2 - sys sys_inotify_init1 1 - sys sys_preadv 6 /* 4330 */ - sys sys_pwritev 6 - sys sys_rt_tgsigqueueinfo 4 - sys sys_perf_event_open 5 - sys sys_accept4 4 - sys sys_recvmmsg 5 /* 4335 */ - sys sys_fanotify_init 2 - sys sys_fanotify_mark 6 - sys sys_prlimit64 4 - sys sys_name_to_handle_at 5 - sys sys_open_by_handle_at 3 /* 4340 */ - sys sys_clock_adjtime 2 - sys sys_syncfs 1 - sys sys_sendmmsg 4 - sys sys_setns 2 - sys sys_process_vm_readv 6 /* 4345 */ - sys sys_process_vm_writev 6 - sys sys_kcmp 5 - sys sys_finit_module 3 - .endm - - /* We pre-compute the number of _instruction_ bytes needed to - load or store the arguments 6-8. Negative values are ignored. */ - - .macro sys function, nargs - PTR \function - LONG (\nargs << 2) - (5 << 2) - .endm - - .align 3 - .type sys_call_table,@object + +#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) +#define __SYSCALL(nr, entry) PTR_WD entry + .align 2 + .type sys_call_table, @object EXPORT(sys_call_table) - syscalltable - .size sys_call_table, . - sys_call_table +#include <asm/syscall_table_o32.h> diff --git a/arch/mips/kernel/scall64-64.S b/arch/mips/kernel/scall64-64.S deleted file mode 100644 index be6627ead619..000000000000 --- a/arch/mips/kernel/scall64-64.S +++ /dev/null @@ -1,427 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 1995, 96, 97, 98, 99, 2000, 01, 02 by Ralf Baechle - * Copyright (C) 1999, 2000 Silicon Graphics, Inc. - * Copyright (C) 2001 MIPS Technologies, Inc. - */ -#include <linux/errno.h> -#include <asm/asm.h> -#include <asm/asmmacro.h> -#include <asm/irqflags.h> -#include <asm/mipsregs.h> -#include <asm/regdef.h> -#include <asm/stackframe.h> -#include <asm/asm-offsets.h> -#include <asm/sysmips.h> -#include <asm/thread_info.h> -#include <asm/unistd.h> -#include <asm/war.h> - -#ifndef CONFIG_BINFMT_ELF32 -/* Neither O32 nor N32, so define handle_sys here */ -#define handle_sys64 handle_sys -#endif - - .align 5 -NESTED(handle_sys64, PT_SIZE, sp) -#if !defined(CONFIG_MIPS32_O32) && !defined(CONFIG_MIPS32_N32) - /* - * When 32-bit compatibility is configured scall_o32.S - * already did this. - */ - .set noat - SAVE_SOME - TRACE_IRQS_ON_RELOAD - STI - .set at -#endif - - dsubu t0, v0, __NR_64_Linux # check syscall number - sltiu t0, t0, __NR_64_Linux_syscalls + 1 -#if !defined(CONFIG_MIPS32_O32) && !defined(CONFIG_MIPS32_N32) - ld t1, PT_EPC(sp) # skip syscall on return - daddiu t1, 4 # skip to next instruction - sd t1, PT_EPC(sp) -#endif - beqz t0, illegal_syscall - - dsll t0, v0, 3 # offset into table - ld t2, (sys_call_table - (__NR_64_Linux * 8))(t0) - # syscall routine - - sd a3, PT_R26(sp) # save a3 for syscall restarting - - li t1, _TIF_WORK_SYSCALL_ENTRY - LONG_L t0, TI_FLAGS($28) # syscall tracing enabled? - and t0, t1, t0 - bnez t0, syscall_trace_entry - - jalr t2 # Do The Real Thing (TM) - - li t0, -EMAXERRNO - 1 # error? - sltu t0, t0, v0 - sd t0, PT_R7(sp) # set error flag - beqz t0, 1f - - ld t1, PT_R2(sp) # syscall number - dnegu v0 # error - sd t1, PT_R0(sp) # save it for syscall restarting -1: sd v0, PT_R2(sp) # result - -n64_syscall_exit: - j syscall_exit_partial - -/* ------------------------------------------------------------------------ */ - -syscall_trace_entry: - SAVE_STATIC - move s0, t2 - move a0, sp - jal syscall_trace_enter - - move t0, s0 - RESTORE_STATIC - ld a0, PT_R4(sp) # Restore argument registers - ld a1, PT_R5(sp) - ld a2, PT_R6(sp) - ld a3, PT_R7(sp) - ld a4, PT_R8(sp) - ld a5, PT_R9(sp) - jalr t0 - - li t0, -EMAXERRNO - 1 # error? - sltu t0, t0, v0 - sd t0, PT_R7(sp) # set error flag - beqz t0, 1f - - ld t1, PT_R2(sp) # syscall number - dnegu v0 # error - sd t1, PT_R0(sp) # save it for syscall restarting -1: sd v0, PT_R2(sp) # result - - j syscall_exit - -illegal_syscall: - /* This also isn't a 64-bit syscall, throw an error. */ - li v0, ENOSYS # error - sd v0, PT_R2(sp) - li t0, 1 # set error flag - sd t0, PT_R7(sp) - j n64_syscall_exit - END(handle_sys64) - - .align 3 -sys_call_table: - PTR sys_read /* 5000 */ - PTR sys_write - PTR sys_open - PTR sys_close - PTR sys_newstat - PTR sys_newfstat /* 5005 */ - PTR sys_newlstat - PTR sys_poll - PTR sys_lseek - PTR sys_mips_mmap - PTR sys_mprotect /* 5010 */ - PTR sys_munmap - PTR sys_brk - PTR sys_rt_sigaction - PTR sys_rt_sigprocmask - PTR sys_ioctl /* 5015 */ - PTR sys_pread64 - PTR sys_pwrite64 - PTR sys_readv - PTR sys_writev - PTR sys_access /* 5020 */ - PTR sysm_pipe - PTR sys_select - PTR sys_sched_yield - PTR sys_mremap - PTR sys_msync /* 5025 */ - PTR sys_mincore - PTR sys_madvise - PTR sys_shmget - PTR sys_shmat - PTR sys_shmctl /* 5030 */ - PTR sys_dup - PTR sys_dup2 - PTR sys_pause - PTR sys_nanosleep - PTR sys_getitimer /* 5035 */ - PTR sys_setitimer - PTR sys_alarm - PTR sys_getpid - PTR sys_sendfile64 - PTR sys_socket /* 5040 */ - PTR sys_connect - PTR sys_accept - PTR sys_sendto - PTR sys_recvfrom - PTR sys_sendmsg /* 5045 */ - PTR sys_recvmsg - PTR sys_shutdown - PTR sys_bind - PTR sys_listen - PTR sys_getsockname /* 5050 */ - PTR sys_getpeername - PTR sys_socketpair - PTR sys_setsockopt - PTR sys_getsockopt - PTR __sys_clone /* 5055 */ - PTR __sys_fork - PTR sys_execve - PTR sys_exit - PTR sys_wait4 - PTR sys_kill /* 5060 */ - PTR sys_newuname - PTR sys_semget - PTR sys_semop - PTR sys_semctl - PTR sys_shmdt /* 5065 */ - PTR sys_msgget - PTR sys_msgsnd - PTR sys_msgrcv - PTR sys_msgctl - PTR sys_fcntl /* 5070 */ - PTR sys_flock - PTR sys_fsync - PTR sys_fdatasync - PTR sys_truncate - PTR sys_ftruncate /* 5075 */ - PTR sys_getdents - PTR sys_getcwd - PTR sys_chdir - PTR sys_fchdir - PTR sys_rename /* 5080 */ - PTR sys_mkdir - PTR sys_rmdir - PTR sys_creat - PTR sys_link - PTR sys_unlink /* 5085 */ - PTR sys_symlink - PTR sys_readlink - PTR sys_chmod - PTR sys_fchmod - PTR sys_chown /* 5090 */ - PTR sys_fchown - PTR sys_lchown - PTR sys_umask - PTR sys_gettimeofday - PTR sys_getrlimit /* 5095 */ - PTR sys_getrusage - PTR sys_sysinfo - PTR sys_times - PTR sys_ptrace - PTR sys_getuid /* 5100 */ - PTR sys_syslog - PTR sys_getgid - PTR sys_setuid - PTR sys_setgid - PTR sys_geteuid /* 5105 */ - PTR sys_getegid - PTR sys_setpgid - PTR sys_getppid - PTR sys_getpgrp - PTR sys_setsid /* 5110 */ - PTR sys_setreuid - PTR sys_setregid - PTR sys_getgroups - PTR sys_setgroups - PTR sys_setresuid /* 5115 */ - PTR sys_getresuid - PTR sys_setresgid - PTR sys_getresgid - PTR sys_getpgid - PTR sys_setfsuid /* 5120 */ - PTR sys_setfsgid - PTR sys_getsid - PTR sys_capget - PTR sys_capset - PTR sys_rt_sigpending /* 5125 */ - PTR sys_rt_sigtimedwait - PTR sys_rt_sigqueueinfo - PTR sys_rt_sigsuspend - PTR sys_sigaltstack - PTR sys_utime /* 5130 */ - PTR sys_mknod - PTR sys_personality - PTR sys_ustat - PTR sys_statfs - PTR sys_fstatfs /* 5135 */ - PTR sys_sysfs - PTR sys_getpriority - PTR sys_setpriority - PTR sys_sched_setparam - PTR sys_sched_getparam /* 5140 */ - PTR sys_sched_setscheduler - PTR sys_sched_getscheduler - PTR sys_sched_get_priority_max - PTR sys_sched_get_priority_min - PTR sys_sched_rr_get_interval /* 5145 */ - PTR sys_mlock - PTR sys_munlock - PTR sys_mlockall - PTR sys_munlockall - PTR sys_vhangup /* 5150 */ - PTR sys_pivot_root - PTR sys_sysctl - PTR sys_prctl - PTR sys_adjtimex - PTR sys_setrlimit /* 5155 */ - PTR sys_chroot - PTR sys_sync - PTR sys_acct - PTR sys_settimeofday - PTR sys_mount /* 5160 */ - PTR sys_umount - PTR sys_swapon - PTR sys_swapoff - PTR sys_reboot - PTR sys_sethostname /* 5165 */ - PTR sys_setdomainname - PTR sys_ni_syscall /* was create_module */ - PTR sys_init_module - PTR sys_delete_module - PTR sys_ni_syscall /* 5170, was get_kernel_syms */ - PTR sys_ni_syscall /* was query_module */ - PTR sys_quotactl - PTR sys_ni_syscall /* was nfsservctl */ - PTR sys_ni_syscall /* res. for getpmsg */ - PTR sys_ni_syscall /* 5175 for putpmsg */ - PTR sys_ni_syscall /* res. for afs_syscall */ - PTR sys_ni_syscall /* res. for security */ - PTR sys_gettid - PTR sys_readahead - PTR sys_setxattr /* 5180 */ - PTR sys_lsetxattr - PTR sys_fsetxattr - PTR sys_getxattr - PTR sys_lgetxattr - PTR sys_fgetxattr /* 5185 */ - PTR sys_listxattr - PTR sys_llistxattr - PTR sys_flistxattr - PTR sys_removexattr - PTR sys_lremovexattr /* 5190 */ - PTR sys_fremovexattr - PTR sys_tkill - PTR sys_ni_syscall - PTR sys_futex - PTR sys_sched_setaffinity /* 5195 */ - PTR sys_sched_getaffinity - PTR sys_cacheflush - PTR sys_cachectl - PTR sys_sysmips - PTR sys_io_setup /* 5200 */ - PTR sys_io_destroy - PTR sys_io_getevents - PTR sys_io_submit - PTR sys_io_cancel - PTR sys_exit_group /* 5205 */ - PTR sys_lookup_dcookie - PTR sys_epoll_create - PTR sys_epoll_ctl - PTR sys_epoll_wait - PTR sys_remap_file_pages /* 5210 */ - PTR sys_rt_sigreturn - PTR sys_set_tid_address - PTR sys_restart_syscall - PTR sys_semtimedop - PTR sys_fadvise64_64 /* 5215 */ - PTR sys_timer_create - PTR sys_timer_settime - PTR sys_timer_gettime - PTR sys_timer_getoverrun - PTR sys_timer_delete /* 5220 */ - PTR sys_clock_settime - PTR sys_clock_gettime - PTR sys_clock_getres - PTR sys_clock_nanosleep - PTR sys_tgkill /* 5225 */ - PTR sys_utimes - PTR sys_mbind - PTR sys_ni_syscall /* sys_get_mempolicy */ - PTR sys_ni_syscall /* sys_set_mempolicy */ - PTR sys_mq_open /* 5230 */ - PTR sys_mq_unlink - PTR sys_mq_timedsend - PTR sys_mq_timedreceive - PTR sys_mq_notify - PTR sys_mq_getsetattr /* 5235 */ - PTR sys_ni_syscall /* sys_vserver */ - PTR sys_waitid - PTR sys_ni_syscall /* available, was setaltroot */ - PTR sys_add_key - PTR sys_request_key /* 5240 */ - PTR sys_keyctl - PTR sys_set_thread_area - PTR sys_inotify_init - PTR sys_inotify_add_watch - PTR sys_inotify_rm_watch /* 5245 */ - PTR sys_migrate_pages - PTR sys_openat - PTR sys_mkdirat - PTR sys_mknodat - PTR sys_fchownat /* 5250 */ - PTR sys_futimesat - PTR sys_newfstatat - PTR sys_unlinkat - PTR sys_renameat - PTR sys_linkat /* 5255 */ - PTR sys_symlinkat - PTR sys_readlinkat - PTR sys_fchmodat - PTR sys_faccessat - PTR sys_pselect6 /* 5260 */ - PTR sys_ppoll - PTR sys_unshare - PTR sys_splice - PTR sys_sync_file_range - PTR sys_tee /* 5265 */ - PTR sys_vmsplice - PTR sys_move_pages - PTR sys_set_robust_list - PTR sys_get_robust_list - PTR sys_kexec_load /* 5270 */ - PTR sys_getcpu - PTR sys_epoll_pwait - PTR sys_ioprio_set - PTR sys_ioprio_get - PTR sys_utimensat /* 5275 */ - PTR sys_signalfd - PTR sys_ni_syscall /* was timerfd */ - PTR sys_eventfd - PTR sys_fallocate - PTR sys_timerfd_create /* 5280 */ - PTR sys_timerfd_gettime - PTR sys_timerfd_settime - PTR sys_signalfd4 - PTR sys_eventfd2 - PTR sys_epoll_create1 /* 5285 */ - PTR sys_dup3 - PTR sys_pipe2 - PTR sys_inotify_init1 - PTR sys_preadv - PTR sys_pwritev /* 5290 */ - PTR sys_rt_tgsigqueueinfo - PTR sys_perf_event_open - PTR sys_accept4 - PTR sys_recvmmsg - PTR sys_fanotify_init /* 5295 */ - PTR sys_fanotify_mark - PTR sys_prlimit64 - PTR sys_name_to_handle_at - PTR sys_open_by_handle_at - PTR sys_clock_adjtime /* 5300 */ - PTR sys_syncfs - PTR sys_sendmmsg - PTR sys_setns - PTR sys_process_vm_readv - PTR sys_process_vm_writev /* 5305 */ - PTR sys_kcmp - PTR sys_finit_module - PTR sys_getdents64 - .size sys_call_table,.-sys_call_table diff --git a/arch/mips/kernel/scall64-n32.S b/arch/mips/kernel/scall64-n32.S index cab150789c8d..97788859238c 100644 --- a/arch/mips/kernel/scall64-n32.S +++ b/arch/mips/kernel/scall64-n32.S @@ -33,7 +33,7 @@ NESTED(handle_sysn32, PT_SIZE, sp) #endif dsubu t0, v0, __NR_N32_Linux # check syscall number - sltiu t0, t0, __NR_N32_Linux_syscalls + 1 + sltiu t0, t0, __NR_N32_Linux_syscalls #ifndef CONFIG_MIPS32_O32 ld t1, PT_EPC(sp) # skip syscall on return @@ -42,16 +42,19 @@ NESTED(handle_sysn32, PT_SIZE, sp) #endif beqz t0, not_n32_scall - dsll t0, v0, 3 # offset into table - ld t2, (sysn32_call_table - (__NR_N32_Linux * 8))(t0) - sd a3, PT_R26(sp) # save a3 for syscall restarting + LONG_S v0, TI_SYSCALL($28) # Store syscall number + li t1, _TIF_WORK_SYSCALL_ENTRY LONG_L t0, TI_FLAGS($28) # syscall tracing enabled? and t0, t1, t0 bnez t0, n32_syscall_trace_entry +syscall_common: + dsll t0, v0, 3 # offset into table + ld t2, (sysn32_call_table - (__NR_N32_Linux * 8))(t0) + jalr t2 # Do The Real Thing (TM) li t0, -EMAXERRNO - 1 # error? @@ -70,31 +73,27 @@ NESTED(handle_sysn32, PT_SIZE, sp) n32_syscall_trace_entry: SAVE_STATIC - move s0, t2 move a0, sp jal syscall_trace_enter - move t0, s0 + bltz v0, 1f # seccomp failed? Skip syscall + RESTORE_STATIC + ld v0, PT_R2(sp) # Restore syscall (maybe modified) ld a0, PT_R4(sp) # Restore argument registers ld a1, PT_R5(sp) ld a2, PT_R6(sp) ld a3, PT_R7(sp) ld a4, PT_R8(sp) ld a5, PT_R9(sp) - jalr t0 - li t0, -EMAXERRNO - 1 # error? - sltu t0, t0, v0 - sd t0, PT_R7(sp) # set error flag - beqz t0, 1f + dsubu t2, v0, __NR_N32_Linux # check (new) syscall number + sltiu t0, t2, __NR_N32_Linux_syscalls + beqz t0, not_n32_scall - ld t1, PT_R2(sp) # syscall number - dnegu v0 # error - sd t1, PT_R0(sp) # save it for syscall restarting -1: sd v0, PT_R2(sp) # result + j syscall_common - j syscall_exit +1: j syscall_exit not_n32_scall: /* This is not an n32 compatibility syscall, pass it on to @@ -103,318 +102,7 @@ not_n32_scall: END(handle_sysn32) +#define __SYSCALL(nr, entry) PTR_WD entry + .type sysn32_call_table, @object EXPORT(sysn32_call_table) - PTR sys_read /* 6000 */ - PTR sys_write - PTR sys_open - PTR sys_close - PTR sys_newstat - PTR sys_newfstat /* 6005 */ - PTR sys_newlstat - PTR sys_poll - PTR sys_lseek - PTR sys_mips_mmap - PTR sys_mprotect /* 6010 */ - PTR sys_munmap - PTR sys_brk - PTR compat_sys_rt_sigaction - PTR compat_sys_rt_sigprocmask - PTR compat_sys_ioctl /* 6015 */ - PTR sys_pread64 - PTR sys_pwrite64 - PTR compat_sys_readv - PTR compat_sys_writev - PTR sys_access /* 6020 */ - PTR sysm_pipe - PTR compat_sys_select - PTR sys_sched_yield - PTR sys_mremap - PTR sys_msync /* 6025 */ - PTR sys_mincore - PTR sys_madvise - PTR sys_shmget - PTR sys_shmat - PTR compat_sys_shmctl /* 6030 */ - PTR sys_dup - PTR sys_dup2 - PTR sys_pause - PTR compat_sys_nanosleep - PTR compat_sys_getitimer /* 6035 */ - PTR compat_sys_setitimer - PTR sys_alarm - PTR sys_getpid - PTR compat_sys_sendfile - PTR sys_socket /* 6040 */ - PTR sys_connect - PTR sys_accept - PTR sys_sendto - PTR compat_sys_recvfrom - PTR compat_sys_sendmsg /* 6045 */ - PTR compat_sys_recvmsg - PTR sys_shutdown - PTR sys_bind - PTR sys_listen - PTR sys_getsockname /* 6050 */ - PTR sys_getpeername - PTR sys_socketpair - PTR compat_sys_setsockopt - PTR sys_getsockopt - PTR __sys_clone /* 6055 */ - PTR __sys_fork - PTR compat_sys_execve - PTR sys_exit - PTR compat_sys_wait4 - PTR sys_kill /* 6060 */ - PTR sys_newuname - PTR sys_semget - PTR sys_semop - PTR compat_sys_semctl - PTR sys_shmdt /* 6065 */ - PTR sys_msgget - PTR compat_sys_msgsnd - PTR compat_sys_msgrcv - PTR compat_sys_msgctl - PTR compat_sys_fcntl /* 6070 */ - PTR sys_flock - PTR sys_fsync - PTR sys_fdatasync - PTR sys_truncate - PTR sys_ftruncate /* 6075 */ - PTR compat_sys_getdents - PTR sys_getcwd - PTR sys_chdir - PTR sys_fchdir - PTR sys_rename /* 6080 */ - PTR sys_mkdir - PTR sys_rmdir - PTR sys_creat - PTR sys_link - PTR sys_unlink /* 6085 */ - PTR sys_symlink - PTR sys_readlink - PTR sys_chmod - PTR sys_fchmod - PTR sys_chown /* 6090 */ - PTR sys_fchown - PTR sys_lchown - PTR sys_umask - PTR compat_sys_gettimeofday - PTR compat_sys_getrlimit /* 6095 */ - PTR compat_sys_getrusage - PTR compat_sys_sysinfo - PTR compat_sys_times - PTR compat_sys_ptrace - PTR sys_getuid /* 6100 */ - PTR sys_syslog - PTR sys_getgid - PTR sys_setuid - PTR sys_setgid - PTR sys_geteuid /* 6105 */ - PTR sys_getegid - PTR sys_setpgid - PTR sys_getppid - PTR sys_getpgrp - PTR sys_setsid /* 6110 */ - PTR sys_setreuid - PTR sys_setregid - PTR sys_getgroups - PTR sys_setgroups - PTR sys_setresuid /* 6115 */ - PTR sys_getresuid - PTR sys_setresgid - PTR sys_getresgid - PTR sys_getpgid - PTR sys_setfsuid /* 6120 */ - PTR sys_setfsgid - PTR sys_getsid - PTR sys_capget - PTR sys_capset - PTR compat_sys_rt_sigpending /* 6125 */ - PTR compat_sys_rt_sigtimedwait - PTR compat_sys_rt_sigqueueinfo - PTR compat_sys_rt_sigsuspend - PTR compat_sys_sigaltstack - PTR compat_sys_utime /* 6130 */ - PTR sys_mknod - PTR sys_32_personality - PTR compat_sys_ustat - PTR compat_sys_statfs - PTR compat_sys_fstatfs /* 6135 */ - PTR sys_sysfs - PTR sys_getpriority - PTR sys_setpriority - PTR sys_sched_setparam - PTR sys_sched_getparam /* 6140 */ - PTR sys_sched_setscheduler - PTR sys_sched_getscheduler - PTR sys_sched_get_priority_max - PTR sys_sched_get_priority_min - PTR compat_sys_sched_rr_get_interval /* 6145 */ - PTR sys_mlock - PTR sys_munlock - PTR sys_mlockall - PTR sys_munlockall - PTR sys_vhangup /* 6150 */ - PTR sys_pivot_root - PTR compat_sys_sysctl - PTR sys_prctl - PTR compat_sys_adjtimex - PTR compat_sys_setrlimit /* 6155 */ - PTR sys_chroot - PTR sys_sync - PTR sys_acct - PTR compat_sys_settimeofday - PTR compat_sys_mount /* 6160 */ - PTR sys_umount - PTR sys_swapon - PTR sys_swapoff - PTR sys_reboot - PTR sys_sethostname /* 6165 */ - PTR sys_setdomainname - PTR sys_ni_syscall /* was create_module */ - PTR sys_init_module - PTR sys_delete_module - PTR sys_ni_syscall /* 6170, was get_kernel_syms */ - PTR sys_ni_syscall /* was query_module */ - PTR sys_quotactl - PTR sys_ni_syscall /* was nfsservctl */ - PTR sys_ni_syscall /* res. for getpmsg */ - PTR sys_ni_syscall /* 6175 for putpmsg */ - PTR sys_ni_syscall /* res. for afs_syscall */ - PTR sys_ni_syscall /* res. for security */ - PTR sys_gettid - PTR sys_readahead - PTR sys_setxattr /* 6180 */ - PTR sys_lsetxattr - PTR sys_fsetxattr - PTR sys_getxattr - PTR sys_lgetxattr - PTR sys_fgetxattr /* 6185 */ - PTR sys_listxattr - PTR sys_llistxattr - PTR sys_flistxattr - PTR sys_removexattr - PTR sys_lremovexattr /* 6190 */ - PTR sys_fremovexattr - PTR sys_tkill - PTR sys_ni_syscall - PTR compat_sys_futex - PTR compat_sys_sched_setaffinity /* 6195 */ - PTR compat_sys_sched_getaffinity - PTR sys_cacheflush - PTR sys_cachectl - PTR sys_sysmips - PTR compat_sys_io_setup /* 6200 */ - PTR sys_io_destroy - PTR compat_sys_io_getevents - PTR compat_sys_io_submit - PTR sys_io_cancel - PTR sys_exit_group /* 6205 */ - PTR sys_lookup_dcookie - PTR sys_epoll_create - PTR sys_epoll_ctl - PTR sys_epoll_wait - PTR sys_remap_file_pages /* 6210 */ - PTR sysn32_rt_sigreturn - PTR compat_sys_fcntl64 - PTR sys_set_tid_address - PTR sys_restart_syscall - PTR compat_sys_semtimedop /* 6215 */ - PTR sys_fadvise64_64 - PTR compat_sys_statfs64 - PTR compat_sys_fstatfs64 - PTR sys_sendfile64 - PTR compat_sys_timer_create /* 6220 */ - PTR compat_sys_timer_settime - PTR compat_sys_timer_gettime - PTR sys_timer_getoverrun - PTR sys_timer_delete - PTR compat_sys_clock_settime /* 6225 */ - PTR compat_sys_clock_gettime - PTR compat_sys_clock_getres - PTR compat_sys_clock_nanosleep - PTR sys_tgkill - PTR compat_sys_utimes /* 6230 */ - PTR sys_ni_syscall /* sys_mbind */ - PTR sys_ni_syscall /* sys_get_mempolicy */ - PTR sys_ni_syscall /* sys_set_mempolicy */ - PTR compat_sys_mq_open - PTR sys_mq_unlink /* 6235 */ - PTR compat_sys_mq_timedsend - PTR compat_sys_mq_timedreceive - PTR compat_sys_mq_notify - PTR compat_sys_mq_getsetattr - PTR sys_ni_syscall /* 6240, sys_vserver */ - PTR compat_sys_waitid - PTR sys_ni_syscall /* available, was setaltroot */ - PTR sys_add_key - PTR sys_request_key - PTR sys_keyctl /* 6245 */ - PTR sys_set_thread_area - PTR sys_inotify_init - PTR sys_inotify_add_watch - PTR sys_inotify_rm_watch - PTR sys_migrate_pages /* 6250 */ - PTR sys_openat - PTR sys_mkdirat - PTR sys_mknodat - PTR sys_fchownat - PTR compat_sys_futimesat /* 6255 */ - PTR sys_newfstatat - PTR sys_unlinkat - PTR sys_renameat - PTR sys_linkat - PTR sys_symlinkat /* 6260 */ - PTR sys_readlinkat - PTR sys_fchmodat - PTR sys_faccessat - PTR compat_sys_pselect6 - PTR compat_sys_ppoll /* 6265 */ - PTR sys_unshare - PTR sys_splice - PTR sys_sync_file_range - PTR sys_tee - PTR compat_sys_vmsplice /* 6270 */ - PTR sys_move_pages - PTR compat_sys_set_robust_list - PTR compat_sys_get_robust_list - PTR compat_sys_kexec_load - PTR sys_getcpu /* 6275 */ - PTR compat_sys_epoll_pwait - PTR sys_ioprio_set - PTR sys_ioprio_get - PTR compat_sys_utimensat - PTR compat_sys_signalfd /* 6280 */ - PTR sys_ni_syscall /* was timerfd */ - PTR sys_eventfd - PTR sys_fallocate - PTR sys_timerfd_create - PTR compat_sys_timerfd_gettime /* 6285 */ - PTR compat_sys_timerfd_settime - PTR compat_sys_signalfd4 - PTR sys_eventfd2 - PTR sys_epoll_create1 - PTR sys_dup3 /* 6290 */ - PTR sys_pipe2 - PTR sys_inotify_init1 - PTR compat_sys_preadv - PTR compat_sys_pwritev - PTR compat_sys_rt_tgsigqueueinfo /* 6295 */ - PTR sys_perf_event_open - PTR sys_accept4 - PTR compat_sys_recvmmsg - PTR sys_getdents64 - PTR sys_fanotify_init /* 6300 */ - PTR sys_fanotify_mark - PTR sys_prlimit64 - PTR sys_name_to_handle_at - PTR sys_open_by_handle_at - PTR compat_sys_clock_adjtime /* 6305 */ - PTR sys_syncfs - PTR compat_sys_sendmmsg - PTR sys_setns - PTR compat_sys_process_vm_readv - PTR compat_sys_process_vm_writev /* 6310 */ - PTR sys_kcmp - PTR sys_finit_module - .size sysn32_call_table,.-sysn32_call_table +#include <asm/syscall_table_n32.h> diff --git a/arch/mips/kernel/scall64-n64.S b/arch/mips/kernel/scall64-n64.S new file mode 100644 index 000000000000..be11ea5cc67e --- /dev/null +++ b/arch/mips/kernel/scall64-n64.S @@ -0,0 +1,116 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1995, 96, 97, 98, 99, 2000, 01, 02 by Ralf Baechle + * Copyright (C) 1999, 2000 Silicon Graphics, Inc. + * Copyright (C) 2001 MIPS Technologies, Inc. + */ +#include <linux/errno.h> +#include <asm/asm.h> +#include <asm/asmmacro.h> +#include <asm/irqflags.h> +#include <asm/mipsregs.h> +#include <asm/regdef.h> +#include <asm/stackframe.h> +#include <asm/asm-offsets.h> +#include <asm/sysmips.h> +#include <asm/thread_info.h> +#include <asm/unistd.h> + +#ifndef CONFIG_MIPS32_COMPAT +/* Neither O32 nor N32, so define handle_sys here */ +#define handle_sys64 handle_sys +#endif + + .align 5 +NESTED(handle_sys64, PT_SIZE, sp) +#if !defined(CONFIG_MIPS32_O32) && !defined(CONFIG_MIPS32_N32) + /* + * When 32-bit compatibility is configured scall_o32.S + * already did this. + */ + .set noat + SAVE_SOME + TRACE_IRQS_ON_RELOAD + STI + .set at +#endif + +#if !defined(CONFIG_MIPS32_O32) && !defined(CONFIG_MIPS32_N32) + ld t1, PT_EPC(sp) # skip syscall on return + daddiu t1, 4 # skip to next instruction + sd t1, PT_EPC(sp) +#endif + + sd a3, PT_R26(sp) # save a3 for syscall restarting + + LONG_S v0, TI_SYSCALL($28) # Store syscall number + + li t1, _TIF_WORK_SYSCALL_ENTRY + LONG_L t0, TI_FLAGS($28) # syscall tracing enabled? + and t0, t1, t0 + bnez t0, syscall_trace_entry + +syscall_common: + dsubu t2, v0, __NR_64_Linux + sltiu t0, t2, __NR_64_Linux_syscalls + beqz t0, illegal_syscall + + dsll t0, t2, 3 # offset into table + dla t2, sys_call_table + daddu t0, t2, t0 + ld t2, (t0) # syscall routine + beqz t2, illegal_syscall + + jalr t2 # Do The Real Thing (TM) + + li t0, -EMAXERRNO - 1 # error? + sltu t0, t0, v0 + sd t0, PT_R7(sp) # set error flag + beqz t0, 1f + + ld t1, PT_R2(sp) # syscall number + dnegu v0 # error + sd t1, PT_R0(sp) # save it for syscall restarting +1: sd v0, PT_R2(sp) # result + +n64_syscall_exit: + j syscall_exit_partial + +/* ------------------------------------------------------------------------ */ + +syscall_trace_entry: + SAVE_STATIC + move a0, sp + jal syscall_trace_enter + + bltz v0, 1f # seccomp failed? Skip syscall + + RESTORE_STATIC + ld v0, PT_R2(sp) # Restore syscall (maybe modified) + ld a0, PT_R4(sp) # Restore argument registers + ld a1, PT_R5(sp) + ld a2, PT_R6(sp) + ld a3, PT_R7(sp) + ld a4, PT_R8(sp) + ld a5, PT_R9(sp) + j syscall_common + +1: j syscall_exit + +illegal_syscall: + /* This also isn't a 64-bit syscall, throw an error. */ + li v0, ENOSYS # error + sd v0, PT_R2(sp) + li t0, 1 # set error flag + sd t0, PT_R7(sp) + j n64_syscall_exit + END(handle_sys64) + +#define __SYSCALL(nr, entry) PTR_WD entry + .align 3 + .type sys_call_table, @object +EXPORT(sys_call_table) +#include <asm/syscall_table_n64.h> diff --git a/arch/mips/kernel/scall64-o32.S b/arch/mips/kernel/scall64-o32.S index 37605dc8eef7..7a5abb73e531 100644 --- a/arch/mips/kernel/scall64-o32.S +++ b/arch/mips/kernel/scall64-o32.S @@ -34,14 +34,14 @@ NESTED(handle_sys, PT_SIZE, sp) ld t1, PT_EPC(sp) # skip syscall on return dsubu t0, v0, __NR_O32_Linux # check syscall number - sltiu t0, t0, __NR_O32_Linux_syscalls + 1 + sltiu t0, t0, __NR_O32_Linux_syscalls daddiu t1, 4 # skip to next instruction sd t1, PT_EPC(sp) beqz t0, not_o32_scall #if 0 SAVE_ALL move a1, v0 - PRINT("Scall %ld\n") + ASM_PRINT("Scall %ld\n") RESTORE_ALL #endif @@ -52,9 +52,6 @@ NESTED(handle_sys, PT_SIZE, sp) sll a2, a2, 0 sll a3, a3, 0 - dsll t0, v0, 3 # offset into table - ld t2, (sys_call_table - (__NR_O32_Linux * 8))(t0) - sd a3, PT_R26(sp) # save a3 for syscall restarting /* @@ -69,23 +66,44 @@ NESTED(handle_sys, PT_SIZE, sp) daddu t1, t0, 32 bltz t1, bad_stack -1: lw a4, 16(t0) # argument #5 from usp -2: lw a5, 20(t0) # argument #6 from usp -3: lw a6, 24(t0) # argument #7 from usp -4: lw a7, 28(t0) # argument #8 from usp (for indirect syscalls) +load_a4: lw a4, 16(t0) # argument #5 from usp +load_a5: lw a5, 20(t0) # argument #6 from usp +load_a6: lw a6, 24(t0) # argument #7 from usp +load_a7: lw a7, 28(t0) # argument #8 from usp +loads_done: .section __ex_table,"a" - PTR 1b, bad_stack - PTR 2b, bad_stack - PTR 3b, bad_stack - PTR 4b, bad_stack + PTR_WD load_a4, bad_stack_a4 + PTR_WD load_a5, bad_stack_a5 + PTR_WD load_a6, bad_stack_a6 + PTR_WD load_a7, bad_stack_a7 .previous + /* + * absolute syscall number is in v0 unless we called syscall(__NR_###) + * where the real syscall number is in a0 + * note: NR_syscall is the first O32 syscall but the macro is + * only defined when compiling with -mabi=32 (CONFIG_32BIT) + * therefore __NR_O32_Linux is used (4000) + */ + + subu t2, v0, __NR_O32_Linux + bnez t2, 1f /* __NR_syscall at offset 0 */ + LONG_S a0, TI_SYSCALL($28) # Save a0 as syscall number + b 2f +1: + LONG_S v0, TI_SYSCALL($28) # Save v0 as syscall number +2: + li t1, _TIF_WORK_SYSCALL_ENTRY LONG_L t0, TI_FLAGS($28) # syscall tracing enabled? and t0, t1, t0 bnez t0, trace_a_syscall +syscall_common: + dsll t0, v0, 3 # offset into table + ld t2, (sys32_call_table - (__NR_O32_Linux * 8))(t0) + jalr t2 # Do The Real Thing (TM) li t0, -EMAXERRNO - 1 # error? @@ -110,12 +128,13 @@ trace_a_syscall: sd a6, PT_R10(sp) sd a7, PT_R11(sp) # For indirect syscalls - move s0, t2 # Save syscall pointer move a0, sp jal syscall_trace_enter - move t0, s0 + bltz v0, 1f # seccomp failed? Skip syscall + RESTORE_STATIC + ld v0, PT_R2(sp) # Restore syscall (maybe modified) ld a0, PT_R4(sp) # Restore argument registers ld a1, PT_R5(sp) ld a2, PT_R6(sp) @@ -124,19 +143,14 @@ trace_a_syscall: ld a5, PT_R9(sp) ld a6, PT_R10(sp) ld a7, PT_R11(sp) # For indirect syscalls - jalr t0 - li t0, -EMAXERRNO - 1 # error? - sltu t0, t0, v0 - sd t0, PT_R7(sp) # set error flag - beqz t0, 1f + dsubu t0, v0, __NR_O32_Linux # check (new) syscall number + sltiu t0, t0, __NR_O32_Linux_syscalls + beqz t0, not_o32_scall - ld t1, PT_R2(sp) # syscall number - dnegu v0 # error - sd t1, PT_R0(sp) # save it for syscall restarting -1: sd v0, PT_R2(sp) # result + j syscall_common - j syscall_exit +1: j syscall_exit /* ------------------------------------------------------------------------ */ @@ -150,6 +164,22 @@ bad_stack: sd t0, PT_R7(sp) j o32_syscall_exit +bad_stack_a4: + li a4, 0 + b load_a5 + +bad_stack_a5: + li a5, 0 + b load_a6 + +bad_stack_a6: + li a6, 0 + b load_a7 + +bad_stack_a7: + li a7, 0 + b loads_done + not_o32_scall: /* * This is not an o32 compatibility syscall, pass it on @@ -164,11 +194,11 @@ not_o32_scall: LEAF(sys32_syscall) subu t0, a0, __NR_O32_Linux # check syscall number - sltiu v0, t0, __NR_O32_Linux_syscalls + 1 + sltiu v0, t0, __NR_O32_Linux_syscalls beqz t0, einval # do not recurse dsll t1, t0, 3 beqz v0, einval - ld t2, sys_call_table(t1) # syscall routine + ld t2, sys32_call_table(t1) # syscall routine move a0, a1 # shift argument registers move a1, a2 @@ -177,11 +207,6 @@ LEAF(sys32_syscall) move a4, a5 move a5, a6 move a6, a7 - sd a0, PT_R4(sp) # ... and push back a0 - a3, some - sd a1, PT_R5(sp) # syscalls expect them there - sd a2, PT_R6(sp) - sd a3, PT_R7(sp) - sd a3, PT_R26(sp) # update a3 for syscall restarting jr t2 /* Unreached */ @@ -189,356 +214,9 @@ einval: li v0, -ENOSYS jr ra END(sys32_syscall) +#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, compat) +#define __SYSCALL(nr, entry) PTR_WD entry .align 3 - .type sys_call_table,@object -sys_call_table: - PTR sys32_syscall /* 4000 */ - PTR sys_exit - PTR __sys_fork - PTR sys_read - PTR sys_write - PTR compat_sys_open /* 4005 */ - PTR sys_close - PTR sys_waitpid - PTR sys_creat - PTR sys_link - PTR sys_unlink /* 4010 */ - PTR compat_sys_execve - PTR sys_chdir - PTR compat_sys_time - PTR sys_mknod - PTR sys_chmod /* 4015 */ - PTR sys_lchown - PTR sys_ni_syscall - PTR sys_ni_syscall /* was sys_stat */ - PTR sys_lseek - PTR sys_getpid /* 4020 */ - PTR compat_sys_mount - PTR sys_oldumount - PTR sys_setuid - PTR sys_getuid - PTR compat_sys_stime /* 4025 */ - PTR compat_sys_ptrace - PTR sys_alarm - PTR sys_ni_syscall /* was sys_fstat */ - PTR sys_pause - PTR compat_sys_utime /* 4030 */ - PTR sys_ni_syscall - PTR sys_ni_syscall - PTR sys_access - PTR sys_nice - PTR sys_ni_syscall /* 4035 */ - PTR sys_sync - PTR sys_kill - PTR sys_rename - PTR sys_mkdir - PTR sys_rmdir /* 4040 */ - PTR sys_dup - PTR sysm_pipe - PTR compat_sys_times - PTR sys_ni_syscall - PTR sys_brk /* 4045 */ - PTR sys_setgid - PTR sys_getgid - PTR sys_ni_syscall /* was signal 2 */ - PTR sys_geteuid - PTR sys_getegid /* 4050 */ - PTR sys_acct - PTR sys_umount - PTR sys_ni_syscall - PTR compat_sys_ioctl - PTR compat_sys_fcntl /* 4055 */ - PTR sys_ni_syscall - PTR sys_setpgid - PTR sys_ni_syscall - PTR sys_olduname - PTR sys_umask /* 4060 */ - PTR sys_chroot - PTR compat_sys_ustat - PTR sys_dup2 - PTR sys_getppid - PTR sys_getpgrp /* 4065 */ - PTR sys_setsid - PTR sys_32_sigaction - PTR sys_sgetmask - PTR sys_ssetmask - PTR sys_setreuid /* 4070 */ - PTR sys_setregid - PTR sys32_sigsuspend - PTR compat_sys_sigpending - PTR sys_sethostname - PTR compat_sys_setrlimit /* 4075 */ - PTR compat_sys_getrlimit - PTR compat_sys_getrusage - PTR compat_sys_gettimeofday - PTR compat_sys_settimeofday - PTR sys_getgroups /* 4080 */ - PTR sys_setgroups - PTR sys_ni_syscall /* old_select */ - PTR sys_symlink - PTR sys_ni_syscall /* was sys_lstat */ - PTR sys_readlink /* 4085 */ - PTR sys_uselib - PTR sys_swapon - PTR sys_reboot - PTR compat_sys_old_readdir - PTR sys_mips_mmap /* 4090 */ - PTR sys_munmap - PTR compat_sys_truncate - PTR compat_sys_ftruncate - PTR sys_fchmod - PTR sys_fchown /* 4095 */ - PTR sys_getpriority - PTR sys_setpriority - PTR sys_ni_syscall - PTR compat_sys_statfs - PTR compat_sys_fstatfs /* 4100 */ - PTR sys_ni_syscall /* sys_ioperm */ - PTR compat_sys_socketcall - PTR sys_syslog - PTR compat_sys_setitimer - PTR compat_sys_getitimer /* 4105 */ - PTR compat_sys_newstat - PTR compat_sys_newlstat - PTR compat_sys_newfstat - PTR sys_uname - PTR sys_ni_syscall /* sys_ioperm *//* 4110 */ - PTR sys_vhangup - PTR sys_ni_syscall /* was sys_idle */ - PTR sys_ni_syscall /* sys_vm86 */ - PTR compat_sys_wait4 - PTR sys_swapoff /* 4115 */ - PTR compat_sys_sysinfo - PTR compat_sys_ipc - PTR sys_fsync - PTR sys32_sigreturn - PTR __sys_clone /* 4120 */ - PTR sys_setdomainname - PTR sys_newuname - PTR sys_ni_syscall /* sys_modify_ldt */ - PTR compat_sys_adjtimex - PTR sys_mprotect /* 4125 */ - PTR compat_sys_sigprocmask - PTR sys_ni_syscall /* was creat_module */ - PTR sys_init_module - PTR sys_delete_module - PTR sys_ni_syscall /* 4130, get_kernel_syms */ - PTR sys_quotactl - PTR sys_getpgid - PTR sys_fchdir - PTR sys_bdflush - PTR sys_sysfs /* 4135 */ - PTR sys_32_personality - PTR sys_ni_syscall /* for afs_syscall */ - PTR sys_setfsuid - PTR sys_setfsgid - PTR sys_32_llseek /* 4140 */ - PTR compat_sys_getdents - PTR compat_sys_select - PTR sys_flock - PTR sys_msync - PTR compat_sys_readv /* 4145 */ - PTR compat_sys_writev - PTR sys_cacheflush - PTR sys_cachectl - PTR sys_sysmips - PTR sys_ni_syscall /* 4150 */ - PTR sys_getsid - PTR sys_fdatasync - PTR compat_sys_sysctl - PTR sys_mlock - PTR sys_munlock /* 4155 */ - PTR sys_mlockall - PTR sys_munlockall - PTR sys_sched_setparam - PTR sys_sched_getparam - PTR sys_sched_setscheduler /* 4160 */ - PTR sys_sched_getscheduler - PTR sys_sched_yield - PTR sys_sched_get_priority_max - PTR sys_sched_get_priority_min - PTR compat_sys_sched_rr_get_interval /* 4165 */ - PTR compat_sys_nanosleep - PTR sys_mremap - PTR sys_accept - PTR sys_bind - PTR sys_connect /* 4170 */ - PTR sys_getpeername - PTR sys_getsockname - PTR sys_getsockopt - PTR sys_listen - PTR compat_sys_recv /* 4175 */ - PTR compat_sys_recvfrom - PTR compat_sys_recvmsg - PTR sys_send - PTR compat_sys_sendmsg - PTR sys_sendto /* 4180 */ - PTR compat_sys_setsockopt - PTR sys_shutdown - PTR sys_socket - PTR sys_socketpair - PTR sys_setresuid /* 4185 */ - PTR sys_getresuid - PTR sys_ni_syscall /* was query_module */ - PTR sys_poll - PTR sys_ni_syscall /* was nfsservctl */ - PTR sys_setresgid /* 4190 */ - PTR sys_getresgid - PTR sys_prctl - PTR sys32_rt_sigreturn - PTR compat_sys_rt_sigaction - PTR compat_sys_rt_sigprocmask /* 4195 */ - PTR compat_sys_rt_sigpending - PTR compat_sys_rt_sigtimedwait - PTR compat_sys_rt_sigqueueinfo - PTR compat_sys_rt_sigsuspend - PTR sys_32_pread /* 4200 */ - PTR sys_32_pwrite - PTR sys_chown - PTR sys_getcwd - PTR sys_capget - PTR sys_capset /* 4205 */ - PTR compat_sys_sigaltstack - PTR compat_sys_sendfile - PTR sys_ni_syscall - PTR sys_ni_syscall - PTR sys_mips_mmap2 /* 4210 */ - PTR sys_32_truncate64 - PTR sys_32_ftruncate64 - PTR sys_newstat - PTR sys_newlstat - PTR sys_newfstat /* 4215 */ - PTR sys_pivot_root - PTR sys_mincore - PTR sys_madvise - PTR sys_getdents64 - PTR compat_sys_fcntl64 /* 4220 */ - PTR sys_ni_syscall - PTR sys_gettid - PTR sys32_readahead - PTR sys_setxattr - PTR sys_lsetxattr /* 4225 */ - PTR sys_fsetxattr - PTR sys_getxattr - PTR sys_lgetxattr - PTR sys_fgetxattr - PTR sys_listxattr /* 4230 */ - PTR sys_llistxattr - PTR sys_flistxattr - PTR sys_removexattr - PTR sys_lremovexattr - PTR sys_fremovexattr /* 4235 */ - PTR sys_tkill - PTR sys_sendfile64 - PTR compat_sys_futex - PTR compat_sys_sched_setaffinity - PTR compat_sys_sched_getaffinity /* 4240 */ - PTR compat_sys_io_setup - PTR sys_io_destroy - PTR compat_sys_io_getevents - PTR compat_sys_io_submit - PTR sys_io_cancel /* 4245 */ - PTR sys_exit_group - PTR compat_sys_lookup_dcookie - PTR sys_epoll_create - PTR sys_epoll_ctl - PTR sys_epoll_wait /* 4250 */ - PTR sys_remap_file_pages - PTR sys_set_tid_address - PTR sys_restart_syscall - PTR sys32_fadvise64_64 - PTR compat_sys_statfs64 /* 4255 */ - PTR compat_sys_fstatfs64 - PTR compat_sys_timer_create - PTR compat_sys_timer_settime - PTR compat_sys_timer_gettime - PTR sys_timer_getoverrun /* 4260 */ - PTR sys_timer_delete - PTR compat_sys_clock_settime - PTR compat_sys_clock_gettime - PTR compat_sys_clock_getres - PTR compat_sys_clock_nanosleep /* 4265 */ - PTR sys_tgkill - PTR compat_sys_utimes - PTR sys_ni_syscall /* sys_mbind */ - PTR sys_ni_syscall /* sys_get_mempolicy */ - PTR sys_ni_syscall /* 4270 sys_set_mempolicy */ - PTR compat_sys_mq_open - PTR sys_mq_unlink - PTR compat_sys_mq_timedsend - PTR compat_sys_mq_timedreceive - PTR compat_sys_mq_notify /* 4275 */ - PTR compat_sys_mq_getsetattr - PTR sys_ni_syscall /* sys_vserver */ - PTR compat_sys_waitid - PTR sys_ni_syscall /* available, was setaltroot */ - PTR sys_add_key /* 4280 */ - PTR sys_request_key - PTR sys_keyctl - PTR sys_set_thread_area - PTR sys_inotify_init - PTR sys_inotify_add_watch /* 4285 */ - PTR sys_inotify_rm_watch - PTR sys_migrate_pages - PTR compat_sys_openat - PTR sys_mkdirat - PTR sys_mknodat /* 4290 */ - PTR sys_fchownat - PTR compat_sys_futimesat - PTR sys_newfstatat - PTR sys_unlinkat - PTR sys_renameat /* 4295 */ - PTR sys_linkat - PTR sys_symlinkat - PTR sys_readlinkat - PTR sys_fchmodat - PTR sys_faccessat /* 4300 */ - PTR compat_sys_pselect6 - PTR compat_sys_ppoll - PTR sys_unshare - PTR sys_splice - PTR sys32_sync_file_range /* 4305 */ - PTR sys_tee - PTR compat_sys_vmsplice - PTR compat_sys_move_pages - PTR compat_sys_set_robust_list - PTR compat_sys_get_robust_list /* 4310 */ - PTR compat_sys_kexec_load - PTR sys_getcpu - PTR compat_sys_epoll_pwait - PTR sys_ioprio_set - PTR sys_ioprio_get /* 4315 */ - PTR compat_sys_utimensat - PTR compat_sys_signalfd - PTR sys_ni_syscall /* was timerfd */ - PTR sys_eventfd - PTR sys32_fallocate /* 4320 */ - PTR sys_timerfd_create - PTR compat_sys_timerfd_gettime - PTR compat_sys_timerfd_settime - PTR compat_sys_signalfd4 - PTR sys_eventfd2 /* 4325 */ - PTR sys_epoll_create1 - PTR sys_dup3 - PTR sys_pipe2 - PTR sys_inotify_init1 - PTR compat_sys_preadv /* 4330 */ - PTR compat_sys_pwritev - PTR compat_sys_rt_tgsigqueueinfo - PTR sys_perf_event_open - PTR sys_accept4 - PTR compat_sys_recvmmsg /* 4335 */ - PTR sys_fanotify_init - PTR compat_sys_fanotify_mark - PTR sys_prlimit64 - PTR sys_name_to_handle_at - PTR compat_sys_open_by_handle_at /* 4340 */ - PTR compat_sys_clock_adjtime - PTR sys_syncfs - PTR compat_sys_sendmmsg - PTR sys_setns - PTR compat_sys_process_vm_readv /* 4345 */ - PTR compat_sys_process_vm_writev - PTR sys_kcmp - PTR sys_finit_module - .size sys_call_table,.-sys_call_table + .type sys32_call_table,@object +EXPORT(sys32_call_table) +#include <asm/syscall_table_o32.h> diff --git a/arch/mips/kernel/segment.c b/arch/mips/kernel/segment.c new file mode 100644 index 000000000000..24560501c70d --- /dev/null +++ b/arch/mips/kernel/segment.c @@ -0,0 +1,93 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2013 Imagination Technologies Ltd. + */ + +#include <linux/kernel.h> +#include <linux/debugfs.h> +#include <linux/seq_file.h> +#include <asm/cpu.h> +#include <asm/debug.h> +#include <asm/mipsregs.h> + +static void build_segment_config(char *str, unsigned int cfg) +{ + unsigned int am; + static const char * const am_str[] = { + "UK", "MK", "MSK", "MUSK", "MUSUK", "USK", + "RSRVD", "UUSK"}; + + /* Segment access mode. */ + am = (cfg & MIPS_SEGCFG_AM) >> MIPS_SEGCFG_AM_SHIFT; + str += sprintf(str, "%-5s", am_str[am]); + + /* + * Access modes MK, MSK and MUSK are mapped segments. Therefore + * there is no direct physical address mapping unless it becomes + * unmapped uncached at error level due to EU. + */ + if ((am == 0) || (am > 3) || (cfg & MIPS_SEGCFG_EU)) + str += sprintf(str, " %03lx", + ((cfg & MIPS_SEGCFG_PA) >> MIPS_SEGCFG_PA_SHIFT)); + else + str += sprintf(str, " UND"); + + if ((am == 0) || (am > 3)) + str += sprintf(str, " %01ld", + ((cfg & MIPS_SEGCFG_C) >> MIPS_SEGCFG_C_SHIFT)); + else + str += sprintf(str, " U"); + + /* Exception configuration. */ + str += sprintf(str, " %01ld\n", + ((cfg & MIPS_SEGCFG_EU) >> MIPS_SEGCFG_EU_SHIFT)); +} + +static int segments_show(struct seq_file *m, void *v) +{ + unsigned int segcfg; + char str[42]; + + seq_puts(m, "Segment Virtual Size Access Mode Physical Caching EU\n"); + seq_puts(m, "------- ------- ---- ----------- -------- ------- --\n"); + + segcfg = read_c0_segctl0(); + build_segment_config(str, segcfg); + seq_printf(m, " 0 e0000000 512M %s", str); + + segcfg >>= 16; + build_segment_config(str, segcfg); + seq_printf(m, " 1 c0000000 512M %s", str); + + segcfg = read_c0_segctl1(); + build_segment_config(str, segcfg); + seq_printf(m, " 2 a0000000 512M %s", str); + + segcfg >>= 16; + build_segment_config(str, segcfg); + seq_printf(m, " 3 80000000 512M %s", str); + + segcfg = read_c0_segctl2(); + build_segment_config(str, segcfg); + seq_printf(m, " 4 40000000 1G %s", str); + + segcfg >>= 16; + build_segment_config(str, segcfg); + seq_printf(m, " 5 00000000 1G %s\n", str); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(segments); + +static int __init segments_info(void) +{ + if (cpu_has_segments) + debugfs_create_file("segments", S_IRUGO, mips_debugfs_dir, NULL, + &segments_fops); + return 0; +} + +device_initcall(segments_info); diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index c7f90519e58c..11b9b6b63e19 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -11,11 +11,11 @@ * Copyright (C) 2000, 2001, 2002, 2007 Maciej W. Rozycki */ #include <linux/init.h> +#include <linux/cpu.h> +#include <linux/delay.h> #include <linux/ioport.h> #include <linux/export.h> -#include <linux/screen_info.h> #include <linux/memblock.h> -#include <linux/bootmem.h> #include <linux/initrd.h> #include <linux/root_dev.h> #include <linux/highmem.h> @@ -24,32 +24,36 @@ #include <linux/debugfs.h> #include <linux/kexec.h> #include <linux/sizes.h> +#include <linux/device.h> +#include <linux/dma-map-ops.h> +#include <linux/decompress/generic.h> +#include <linux/of_fdt.h> +#include <linux/dmi.h> +#include <linux/crash_dump.h> #include <asm/addrspace.h> #include <asm/bootinfo.h> #include <asm/bugs.h> #include <asm/cache.h> +#include <asm/cdmm.h> #include <asm/cpu.h> +#include <asm/debug.h> +#include <asm/mmzone.h> #include <asm/sections.h> #include <asm/setup.h> #include <asm/smp-ops.h> +#include <asm/mips-cps.h> #include <asm/prom.h> +#include <asm/fw/fw.h> + +#ifdef CONFIG_MIPS_ELF_APPENDED_DTB +char __section(".appended_dtb") __appended_dtb[0x100000]; +#endif /* CONFIG_MIPS_ELF_APPENDED_DTB */ struct cpuinfo_mips cpu_data[NR_CPUS] __read_mostly; EXPORT_SYMBOL(cpu_data); -#ifdef CONFIG_VT -struct screen_info screen_info; -#endif - -/* - * Despite it's name this variable is even if we don't have PCI - */ -unsigned int PCI_DMA_BUS_IS_PHYS; - -EXPORT_SYMBOL(PCI_DMA_BUS_IS_PHYS); - /* * Setup information * @@ -59,76 +63,40 @@ unsigned long mips_machtype __read_mostly = MACH_UNKNOWN; EXPORT_SYMBOL(mips_machtype); -struct boot_mem_map boot_mem_map; - static char __initdata command_line[COMMAND_LINE_SIZE]; char __initdata arcs_cmdline[COMMAND_LINE_SIZE]; #ifdef CONFIG_CMDLINE_BOOL -static char __initdata builtin_cmdline[COMMAND_LINE_SIZE] = CONFIG_CMDLINE; +static const char builtin_cmdline[] __initconst = CONFIG_CMDLINE; +#else +static const char builtin_cmdline[] __initconst = ""; #endif /* * mips_io_port_base is the begin of the address space to which x86 style * I/O ports are mapped. */ -const unsigned long mips_io_port_base = -1; +unsigned long mips_io_port_base = -1; EXPORT_SYMBOL(mips_io_port_base); static struct resource code_resource = { .name = "Kernel code", }; static struct resource data_resource = { .name = "Kernel data", }; +static struct resource bss_resource = { .name = "Kernel bss", }; -static void *detect_magic __initdata = detect_memory_region; - -void __init add_memory_region(phys_t start, phys_t size, long type) -{ - int x = boot_mem_map.nr_map; - int i; - - /* Sanity check */ - if (start + size < start) { - pr_warning("Trying to add an invalid memory region, skipped\n"); - return; - } - - /* - * Try to merge with existing entry, if any. - */ - for (i = 0; i < boot_mem_map.nr_map; i++) { - struct boot_mem_map_entry *entry = boot_mem_map.map + i; - unsigned long top; - - if (entry->type != type) - continue; - - if (start + size < entry->addr) - continue; /* no overlap */ +unsigned long __kaslr_offset __ro_after_init; +EXPORT_SYMBOL(__kaslr_offset); - if (entry->addr + entry->size < start) - continue; /* no overlap */ - - top = max(entry->addr + entry->size, start + size); - entry->addr = min(entry->addr, start); - entry->size = top - entry->addr; - - return; - } - - if (boot_mem_map.nr_map == BOOT_MEM_MAP_MAX) { - pr_err("Ooops! Too many entries in the memory map!\n"); - return; - } +static void *detect_magic __initdata = detect_memory_region; - boot_mem_map.map[x].addr = start; - boot_mem_map.map[x].size = size; - boot_mem_map.map[x].type = type; - boot_mem_map.nr_map++; -} +#ifdef CONFIG_MIPS_AUTO_PFN_OFFSET +unsigned long ARCH_PFN_OFFSET; +EXPORT_SYMBOL(ARCH_PFN_OFFSET); +#endif -void __init detect_memory_region(phys_t start, phys_t sz_min, phys_t sz_max) +void __init detect_memory_region(phys_addr_t start, phys_addr_t sz_min, phys_addr_t sz_max) { void *dm = &detect_magic; - phys_t size; + phys_addr_t size; for (size = sz_min; size < sz_max; size <<= 1) { if (!memcmp(dm, dm + size, sizeof(detect_magic))) @@ -141,37 +109,7 @@ void __init detect_memory_region(phys_t start, phys_t sz_min, phys_t sz_max) ((unsigned long long) sz_min) / SZ_1M, ((unsigned long long) sz_max) / SZ_1M); - add_memory_region(start, size, BOOT_MEM_RAM); -} - -static void __init print_memory_map(void) -{ - int i; - const int field = 2 * sizeof(unsigned long); - - for (i = 0; i < boot_mem_map.nr_map; i++) { - printk(KERN_INFO " memory: %0*Lx @ %0*Lx ", - field, (unsigned long long) boot_mem_map.map[i].size, - field, (unsigned long long) boot_mem_map.map[i].addr); - - switch (boot_mem_map.map[i].type) { - case BOOT_MEM_RAM: - printk(KERN_CONT "(usable)\n"); - break; - case BOOT_MEM_INIT_RAM: - printk(KERN_CONT "(usable after init)\n"); - break; - case BOOT_MEM_ROM_DATA: - printk(KERN_CONT "(ROM data)\n"); - break; - case BOOT_MEM_RESERVED: - printk(KERN_CONT "(reserved)\n"); - break; - default: - printk(KERN_CONT "type %lu\n", boot_mem_map.map[i].type); - break; - } - } + memblock_add(start, size); } /* @@ -209,7 +147,7 @@ static unsigned long __init init_initrd(void) /* * Board specific code or command line parser should have * already set up initrd_start and initrd_end. In these cases - * perfom sanity checks and use them if all looks good. + * perform sanity checks and use them if all looks good. */ if (!initrd_start || initrd_end <= initrd_start) goto disable; @@ -218,10 +156,6 @@ static unsigned long __init init_initrd(void) pr_err("initrd start must be page aligned\n"); goto disable; } - if (initrd_start < PAGE_OFFSET) { - pr_err("initrd start < PAGE_OFFSET\n"); - goto disable; - } /* * Sanitize initrd addresses. For example firmware @@ -234,6 +168,11 @@ static unsigned long __init init_initrd(void) initrd_end = (unsigned long)__va(end); initrd_start = (unsigned long)__va(__pa(initrd_start)); + if (initrd_start < PAGE_OFFSET) { + pr_err("initrd start < PAGE_OFFSET\n"); + goto disable; + } + ROOT_DEV = Root_RAM0; return PFN_UP(end); disable: @@ -242,6 +181,35 @@ disable: return 0; } +/* In some conditions (e.g. big endian bootloader with a little endian + kernel), the initrd might appear byte swapped. Try to detect this and + byte swap it if needed. */ +static void __init maybe_bswap_initrd(void) +{ +#if defined(CONFIG_CPU_CAVIUM_OCTEON) + u64 buf; + + /* Check for CPIO signature */ + if (!memcmp((void *)initrd_start, "070701", 6)) + return; + + /* Check for compressed initrd */ + if (decompress_method((unsigned char *)initrd_start, 8, NULL)) + return; + + /* Try again with a byte swapped header */ + buf = swab64p((u64 *)initrd_start); + if (!memcmp(&buf, "070701", 6) || + decompress_method((unsigned char *)(&buf), 8, NULL)) { + unsigned long i; + + pr_info("Byteswapped initrd detected\n"); + for (i = initrd_start; i < ALIGN(initrd_end, 8); i += 8) + swab64s((u64 *)i); + } +#endif +} + static void __init finalize_initrd(void) { unsigned long size = initrd_end - initrd_start; @@ -255,7 +223,9 @@ static void __init finalize_initrd(void) goto disable; } - reserve_bootmem(__pa(initrd_start), size, BOOTMEM_DEFAULT); + maybe_bswap_initrd(); + + memblock_reserve(__pa(initrd_start), size); initrd_below_start_ok = 1; pr_info("Initial ramdisk at: 0x%lx (%lu bytes)\n", @@ -282,7 +252,7 @@ static unsigned long __init init_initrd(void) * Initialize the bootmem allocator. It also setup initrd related data * if needed. */ -#ifdef CONFIG_SGI_IP27 +#if defined(CONFIG_SGI_IP27) || (defined(CONFIG_CPU_LOONGSON64) && defined(CONFIG_NUMA)) static void __init bootmem_init(void) { @@ -294,160 +264,74 @@ static void __init bootmem_init(void) static void __init bootmem_init(void) { - unsigned long reserved_end; - unsigned long mapstart = ~0UL; - unsigned long bootmap_size; + phys_addr_t ramstart, ramend; + unsigned long start, end; int i; - /* - * Init any data related to initrd. It's a nop if INITRD is - * not selected. Once that done we can determine the low bound - * of usable memory. - */ - reserved_end = max(init_initrd(), - (unsigned long) PFN_UP(__pa_symbol(&_end))); - - /* - * max_low_pfn is not a number of pages. The number of pages - * of the system is given by 'max_low_pfn - min_low_pfn'. - */ - min_low_pfn = ~0UL; - max_low_pfn = 0; + ramstart = memblock_start_of_DRAM(); + ramend = memblock_end_of_DRAM(); /* - * Find the highest page frame number we have available. + * Sanity check any INITRD first. We don't take it into account + * for bootmem setup initially, rely on the end-of-kernel-code + * as our memory range starting point. Once bootmem is inited we + * will reserve the area used for the initrd. */ - for (i = 0; i < boot_mem_map.nr_map; i++) { - unsigned long start, end; - - if (boot_mem_map.map[i].type != BOOT_MEM_RAM) - continue; - - start = PFN_UP(boot_mem_map.map[i].addr); - end = PFN_DOWN(boot_mem_map.map[i].addr - + boot_mem_map.map[i].size); - - if (end > max_low_pfn) - max_low_pfn = end; - if (start < min_low_pfn) - min_low_pfn = start; - if (end <= reserved_end) - continue; - if (start >= mapstart) - continue; - mapstart = max(reserved_end, start); - } + init_initrd(); - if (min_low_pfn >= max_low_pfn) - panic("Incorrect memory mapping !!!"); - if (min_low_pfn > ARCH_PFN_OFFSET) { - pr_info("Wasting %lu bytes for tracking %lu unused pages\n", - (min_low_pfn - ARCH_PFN_OFFSET) * sizeof(struct page), - min_low_pfn - ARCH_PFN_OFFSET); - } else if (min_low_pfn < ARCH_PFN_OFFSET) { - pr_info("%lu free pages won't be used\n", - ARCH_PFN_OFFSET - min_low_pfn); - } - min_low_pfn = ARCH_PFN_OFFSET; + /* Reserve memory occupied by kernel. */ + memblock_reserve(__pa_symbol(&_text), + __pa_symbol(&_end) - __pa_symbol(&_text)); - /* - * Determine low and high memory ranges - */ - max_pfn = max_low_pfn; - if (max_low_pfn > PFN_DOWN(HIGHMEM_START)) { -#ifdef CONFIG_HIGHMEM - highstart_pfn = PFN_DOWN(HIGHMEM_START); - highend_pfn = max_low_pfn; -#endif - max_low_pfn = PFN_DOWN(HIGHMEM_START); - } + /* max_low_pfn is not a number of pages but the end pfn of low mem */ +#ifdef CONFIG_MIPS_AUTO_PFN_OFFSET + ARCH_PFN_OFFSET = PFN_UP(ramstart); +#else /* - * Initialize the boot-time allocator with low memory only. + * Reserve any memory between the start of RAM and PHYS_OFFSET */ - bootmap_size = init_bootmem_node(NODE_DATA(0), mapstart, - min_low_pfn, max_low_pfn); - - - for (i = 0; i < boot_mem_map.nr_map; i++) { - unsigned long start, end; - - start = PFN_UP(boot_mem_map.map[i].addr); - end = PFN_DOWN(boot_mem_map.map[i].addr - + boot_mem_map.map[i].size); - - if (start <= min_low_pfn) - start = min_low_pfn; - if (start >= end) - continue; - -#ifndef CONFIG_HIGHMEM - if (end > max_low_pfn) - end = max_low_pfn; - - /* - * ... finally, is the area going away? - */ - if (end <= start) - continue; -#endif + if (ramstart > PHYS_OFFSET) + memblock_reserve(PHYS_OFFSET, ramstart - PHYS_OFFSET); - memblock_add_node(PFN_PHYS(start), PFN_PHYS(end - start), 0); + if (PFN_UP(ramstart) > ARCH_PFN_OFFSET) { + pr_info("Wasting %lu bytes for tracking %lu unused pages\n", + (unsigned long)((PFN_UP(ramstart) - ARCH_PFN_OFFSET) * sizeof(struct page)), + (unsigned long)(PFN_UP(ramstart) - ARCH_PFN_OFFSET)); } +#endif - /* - * Register fully available low RAM pages with the bootmem allocator. - */ - for (i = 0; i < boot_mem_map.nr_map; i++) { - unsigned long start, end, size; - - start = PFN_UP(boot_mem_map.map[i].addr); - end = PFN_DOWN(boot_mem_map.map[i].addr - + boot_mem_map.map[i].size); - - /* - * Reserve usable memory. - */ - switch (boot_mem_map.map[i].type) { - case BOOT_MEM_RAM: - break; - case BOOT_MEM_INIT_RAM: - memory_present(0, start, end); - continue; - default: - /* Not usable memory */ - continue; - } - + min_low_pfn = ARCH_PFN_OFFSET; + max_pfn = PFN_DOWN(ramend); + for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) { /* - * We are rounding up the start address of usable memory - * and at the end of the usable range downwards. + * Skip highmem here so we get an accurate max_low_pfn if low + * memory stops short of high memory. + * If the region overlaps HIGHMEM_START, end is clipped so + * max_pfn excludes the highmem portion. */ - if (start >= max_low_pfn) + if (start >= PFN_DOWN(HIGHMEM_START)) continue; - if (start < reserved_end) - start = reserved_end; + if (end > PFN_DOWN(HIGHMEM_START)) + end = PFN_DOWN(HIGHMEM_START); if (end > max_low_pfn) - end = max_low_pfn; + max_low_pfn = end; + } - /* - * ... finally, is the area going away? - */ - if (end <= start) - continue; - size = end - start; + if (min_low_pfn >= max_low_pfn) + panic("Incorrect memory mapping !!!"); - /* Register lowmem ranges */ - free_bootmem(PFN_PHYS(start), size << PAGE_SHIFT); - memory_present(0, start, end); + if (max_pfn > PFN_DOWN(HIGHMEM_START)) { + max_low_pfn = PFN_DOWN(HIGHMEM_START); +#ifdef CONFIG_HIGHMEM + highstart_pfn = max_low_pfn; + highend_pfn = max_pfn; +#else + max_pfn = max_low_pfn; +#endif } /* - * Reserve the bootmap memory. - */ - reserve_bootmem(PFN_PHYS(mapstart), bootmap_size, BOOTMEM_DEFAULT); - - /* * Reserve initrd memory if needed. */ finalize_initrd(); @@ -455,33 +339,16 @@ static void __init bootmem_init(void) #endif /* CONFIG_SGI_IP27 */ -/* - * arch_mem_init - initialize memory management subsystem - * - * o plat_mem_setup() detects the memory configuration and will record detected - * memory areas using add_memory_region. - * - * At this stage the memory configuration of the system is known to the - * kernel but generic memory management system is still entirely uninitialized. - * - * o bootmem_init() - * o sparse_init() - * o paging_init() - * - * At this stage the bootmem allocator is ready to use. - * - * NOTE: historically plat_mem_setup did the entire platform initialization. - * This was rather impractical because it meant plat_mem_setup had to - * get away without any kind of memory allocator. To keep old code from - * breaking plat_setup was just renamed to plat_setup and a second platform - * initialization hook for anything else was introduced. - */ - static int usermem __initdata; static int __init early_parse_mem(char *p) { - unsigned long start, size; + phys_addr_t start, size; + + if (!p) { + pr_err("mem parameter is empty, do nothing\n"); + return -EINVAL; + } /* * If a user specifies memory size, we @@ -489,145 +356,95 @@ static int __init early_parse_mem(char *p) * size. */ if (usermem == 0) { - boot_mem_map.nr_map = 0; usermem = 1; + memblock_remove(memblock_start_of_DRAM(), + memblock_end_of_DRAM() - memblock_start_of_DRAM()); } start = 0; size = memparse(p, &p); if (*p == '@') start = memparse(p + 1, &p); - add_memory_region(start, size, BOOT_MEM_RAM); + if (IS_ENABLED(CONFIG_NUMA)) + memblock_add_node(start, size, pa_to_nid(start), MEMBLOCK_NONE); + else + memblock_add(start, size); + return 0; } early_param("mem", early_parse_mem); -#ifdef CONFIG_PROC_VMCORE -unsigned long setup_elfcorehdr, setup_elfcorehdr_size; -static int __init early_parse_elfcorehdr(char *p) +static int __init early_parse_memmap(char *p) { - int i; - - setup_elfcorehdr = memparse(p, &p); - - for (i = 0; i < boot_mem_map.nr_map; i++) { - unsigned long start = boot_mem_map.map[i].addr; - unsigned long end = (boot_mem_map.map[i].addr + - boot_mem_map.map[i].size); - if (setup_elfcorehdr >= start && setup_elfcorehdr < end) { - /* - * Reserve from the elf core header to the end of - * the memory segment, that should all be kdump - * reserved memory. - */ - setup_elfcorehdr_size = end - setup_elfcorehdr; - break; - } - } - /* - * If we don't find it in the memory map, then we shouldn't - * have to worry about it, as the new kernel won't use it. - */ - return 0; -} -early_param("elfcorehdr", early_parse_elfcorehdr); -#endif + char *oldp; + u64 start_at, mem_size; -static void __init arch_mem_addpart(phys_t mem, phys_t end, int type) -{ - phys_t size; - int i; + if (!p) + return -EINVAL; - size = end - mem; - if (!size) - return; + if (!strncmp(p, "exactmap", 8)) { + pr_err("\"memmap=exactmap\" invalid on MIPS\n"); + return 0; + } - /* Make sure it is in the boot_mem_map */ - for (i = 0; i < boot_mem_map.nr_map; i++) { - if (mem >= boot_mem_map.map[i].addr && - mem < (boot_mem_map.map[i].addr + - boot_mem_map.map[i].size)) - return; + oldp = p; + mem_size = memparse(p, &p); + if (p == oldp) + return -EINVAL; + + if (*p == '@') { + start_at = memparse(p+1, &p); + memblock_add(start_at, mem_size); + } else if (*p == '#') { + pr_err("\"memmap=nn#ss\" (force ACPI data) invalid on MIPS\n"); + return -EINVAL; + } else if (*p == '$') { + start_at = memparse(p+1, &p); + memblock_add(start_at, mem_size); + memblock_reserve(start_at, mem_size); + } else { + pr_err("\"memmap\" invalid format!\n"); + return -EINVAL; } - add_memory_region(mem, size, type); + + if (*p == '\0') { + usermem = 1; + return 0; + } else + return -EINVAL; } +early_param("memmap", early_parse_memmap); -static void __init arch_mem_init(char **cmdline_p) +static void __init mips_reserve_vmcore(void) { - extern void plat_mem_setup(void); - - /* call board setup routine */ - plat_mem_setup(); - - /* - * Make sure all kernel memory is in the maps. The "UP" and - * "DOWN" are opposite for initdata since if it crosses over - * into another memory section you don't want that to be - * freed when the initdata is freed. - */ - arch_mem_addpart(PFN_DOWN(__pa_symbol(&_text)) << PAGE_SHIFT, - PFN_UP(__pa_symbol(&_edata)) << PAGE_SHIFT, - BOOT_MEM_RAM); - arch_mem_addpart(PFN_UP(__pa_symbol(&__init_begin)) << PAGE_SHIFT, - PFN_DOWN(__pa_symbol(&__init_end)) << PAGE_SHIFT, - BOOT_MEM_INIT_RAM); - - pr_info("Determined physical RAM map:\n"); - print_memory_map(); - -#ifdef CONFIG_CMDLINE_BOOL -#ifdef CONFIG_CMDLINE_OVERRIDE - strlcpy(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE); -#else - if (builtin_cmdline[0]) { - strlcat(arcs_cmdline, " ", COMMAND_LINE_SIZE); - strlcat(arcs_cmdline, builtin_cmdline, COMMAND_LINE_SIZE); +#ifdef CONFIG_PROC_VMCORE + phys_addr_t start, end; + u64 i; + + if (!elfcorehdr_size) { + for_each_mem_range(i, &start, &end) { + if (elfcorehdr_addr >= start && elfcorehdr_addr < end) { + /* + * Reserve from the elf core header to the end of + * the memory segment, that should all be kdump + * reserved memory. + */ + elfcorehdr_size = end - elfcorehdr_addr; + break; + } + } } - strlcpy(boot_command_line, arcs_cmdline, COMMAND_LINE_SIZE); -#endif -#else - strlcpy(boot_command_line, arcs_cmdline, COMMAND_LINE_SIZE); -#endif - strlcpy(command_line, boot_command_line, COMMAND_LINE_SIZE); - *cmdline_p = command_line; - - parse_early_param(); + pr_info("Reserving %ldKB of memory at %ldKB for kdump\n", + (unsigned long)elfcorehdr_size >> 10, (unsigned long)elfcorehdr_addr >> 10); - if (usermem) { - pr_info("User-defined physical RAM map:\n"); - print_memory_map(); - } - - bootmem_init(); -#ifdef CONFIG_PROC_VMCORE - if (setup_elfcorehdr && setup_elfcorehdr_size) { - printk(KERN_INFO "kdump reserved memory at %lx-%lx\n", - setup_elfcorehdr, setup_elfcorehdr_size); - reserve_bootmem(setup_elfcorehdr, setup_elfcorehdr_size, - BOOTMEM_DEFAULT); - } + memblock_reserve(elfcorehdr_addr, elfcorehdr_size); #endif -#ifdef CONFIG_KEXEC - if (crashk_res.start != crashk_res.end) - reserve_bootmem(crashk_res.start, - crashk_res.end - crashk_res.start + 1, - BOOTMEM_DEFAULT); -#endif - device_tree_init(); - sparse_init(); - plat_swiotlb_setup(); - paging_init(); } -#ifdef CONFIG_KEXEC -static inline unsigned long long get_total_mem(void) -{ - unsigned long long total; - - total = max_pfn - min_low_pfn; - return total << PAGE_SHIFT; -} +/* 64M alignment for crash kernel regions */ +#define CRASH_ALIGN SZ_64M +#define CRASH_ADDR_MAX SZ_512M static void __init mips_parse_crashkernel(void) { @@ -635,12 +452,36 @@ static void __init mips_parse_crashkernel(void) unsigned long long crash_size, crash_base; int ret; - total_mem = get_total_mem(); + if (!IS_ENABLED(CONFIG_CRASH_RESERVE)) + return; + + total_mem = memblock_phys_mem_size(); ret = parse_crashkernel(boot_command_line, total_mem, - &crash_size, &crash_base); + &crash_size, &crash_base, + NULL, NULL, NULL); if (ret != 0 || crash_size <= 0) return; + if (crash_base <= 0) { + crash_base = memblock_phys_alloc_range(crash_size, CRASH_ALIGN, + CRASH_ALIGN, + CRASH_ADDR_MAX); + if (!crash_base) { + pr_warn("crashkernel reservation failed - No suitable area found.\n"); + return; + } + } else { + unsigned long long start; + + start = memblock_phys_alloc_range(crash_size, 1, + crash_base, + crash_base + crash_size); + if (start != crash_base) { + pr_warn("Invalid memory region reserved for crash kernel\n"); + return; + } + } + crashk_res.start = crash_base; crashk_res.end = crash_base + crash_size - 1; } @@ -649,26 +490,206 @@ static void __init request_crashkernel(struct resource *res) { int ret; + if (!IS_ENABLED(CONFIG_CRASH_RESERVE)) + return; + + if (crashk_res.start == crashk_res.end) + return; + ret = request_resource(res, &crashk_res); if (!ret) pr_info("Reserving %ldMB of memory at %ldMB for crashkernel\n", - (unsigned long)((crashk_res.end - - crashk_res.start + 1) >> 20), + (unsigned long)(resource_size(&crashk_res) >> 20), (unsigned long)(crashk_res.start >> 20)); } -#else /* !defined(CONFIG_KEXEC) */ -static void __init mips_parse_crashkernel(void) + +static void __init check_kernel_sections_mem(void) { + phys_addr_t start = __pa_symbol(&_text); + phys_addr_t size = __pa_symbol(&_end) - start; + + if (!memblock_is_region_memory(start, size)) { + pr_info("Kernel sections are not in the memory maps\n"); + memblock_add(start, size); + } } -static void __init request_crashkernel(struct resource *res) +static void __init bootcmdline_append(const char *s, size_t max) +{ + if (!s[0] || !max) + return; + + if (boot_command_line[0]) + strlcat(boot_command_line, " ", COMMAND_LINE_SIZE); + + strlcat(boot_command_line, s, max); +} + +#ifdef CONFIG_OF_EARLY_FLATTREE + +static int __init bootcmdline_scan_chosen(unsigned long node, const char *uname, + int depth, void *data) +{ + bool *dt_bootargs = data; + const char *p; + int l; + + if (depth != 1 || !data || + (strcmp(uname, "chosen") != 0 && strcmp(uname, "chosen@0") != 0)) + return 0; + + p = of_get_flat_dt_prop(node, "bootargs", &l); + if (p != NULL && l > 0) { + bootcmdline_append(p, min(l, COMMAND_LINE_SIZE)); + *dt_bootargs = true; + } + + return 1; +} + +#endif /* CONFIG_OF_EARLY_FLATTREE */ + +static void __init bootcmdline_init(void) +{ + bool dt_bootargs = false; + + /* + * If CMDLINE_OVERRIDE is enabled then initializing the command line is + * trivial - we simply use the built-in command line unconditionally & + * unmodified. + */ + if (IS_ENABLED(CONFIG_CMDLINE_OVERRIDE)) { + strscpy(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE); + return; + } + + /* + * If the user specified a built-in command line & + * MIPS_CMDLINE_BUILTIN_EXTEND, then the built-in command line is + * prepended to arguments from the bootloader or DT so we'll copy them + * to the start of boot_command_line here. Otherwise, empty + * boot_command_line to undo anything early_init_dt_scan_chosen() did. + */ + if (IS_ENABLED(CONFIG_MIPS_CMDLINE_BUILTIN_EXTEND)) + strscpy(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE); + else + boot_command_line[0] = 0; + +#ifdef CONFIG_OF_EARLY_FLATTREE + /* + * If we're configured to take boot arguments from DT, look for those + * now. + */ + if (IS_ENABLED(CONFIG_MIPS_CMDLINE_FROM_DTB) || + IS_ENABLED(CONFIG_MIPS_CMDLINE_DTB_EXTEND)) + of_scan_flat_dt(bootcmdline_scan_chosen, &dt_bootargs); +#endif + + /* + * If we didn't get any arguments from DT (regardless of whether that's + * because we weren't configured to look for them, or because we looked + * & found none) then we'll take arguments from the bootloader. + * plat_mem_setup() should have filled arcs_cmdline with arguments from + * the bootloader. + */ + if (IS_ENABLED(CONFIG_MIPS_CMDLINE_DTB_EXTEND) || !dt_bootargs) + bootcmdline_append(arcs_cmdline, COMMAND_LINE_SIZE); + + /* + * If the user specified a built-in command line & we didn't already + * prepend it, we append it to boot_command_line here. + */ + if (IS_ENABLED(CONFIG_CMDLINE_BOOL) && + !IS_ENABLED(CONFIG_MIPS_CMDLINE_BUILTIN_EXTEND)) + bootcmdline_append(builtin_cmdline, COMMAND_LINE_SIZE); +} + +/* + * arch_mem_init - initialize memory management subsystem + * + * o plat_mem_setup() detects the memory configuration and will record detected + * memory areas using memblock_add. + * + * At this stage the memory configuration of the system is known to the + * kernel but generic memory management system is still entirely uninitialized. + * + * o bootmem_init() + * o sparse_init() + * o paging_init() + * o dma_contiguous_reserve() + * + * At this stage the bootmem allocator is ready to use. + * + * NOTE: historically plat_mem_setup did the entire platform initialization. + * This was rather impractical because it meant plat_mem_setup had to + * get away without any kind of memory allocator. To keep old code from + * breaking plat_setup was just renamed to plat_mem_setup and a second platform + * initialization hook for anything else was introduced. + */ +static void __init arch_mem_init(char **cmdline_p) { + /* call board setup routine */ + plat_mem_setup(); + memblock_set_bottom_up(true); + + bootcmdline_init(); + strscpy(command_line, boot_command_line, COMMAND_LINE_SIZE); + *cmdline_p = command_line; + + parse_early_param(); + + if (usermem) + pr_info("User-defined physical RAM map overwrite\n"); + + check_kernel_sections_mem(); + + early_init_fdt_reserve_self(); + early_init_fdt_scan_reserved_mem(); + +#ifndef CONFIG_NUMA + memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0); +#endif + bootmem_init(); + + /* + * Prevent memblock from allocating high memory. + * This cannot be done before max_low_pfn is detected, so up + * to this point is possible to only reserve physical memory + * with memblock_reserve; memblock_alloc* can be used + * only after this point + */ + memblock_set_current_limit(PFN_PHYS(max_low_pfn)); + + mips_reserve_vmcore(); + + mips_parse_crashkernel(); + device_tree_init(); + + /* + * In order to reduce the possibility of kernel panic when failed to + * get IO TLB memory under CONFIG_SWIOTLB, it is better to allocate + * low memory as small as possible before plat_swiotlb_setup(), so + * make sparse_init() using top-down allocation. + */ + memblock_set_bottom_up(false); + sparse_init(); + memblock_set_bottom_up(true); + + plat_swiotlb_setup(); + + dma_contiguous_reserve(PFN_PHYS(max_low_pfn)); + + /* Reserve for hibernation. */ + memblock_reserve(__pa_symbol(&__nosave_begin), + __pa_symbol(&__nosave_end) - __pa_symbol(&__nosave_begin)); + + early_memtest(PFN_PHYS(ARCH_PFN_OFFSET), PFN_PHYS(max_low_pfn)); } -#endif /* !defined(CONFIG_KEXEC) */ static void __init resource_init(void) { - int i; + phys_addr_t start, end; + u64 i; if (UNCAC_BASE != IO_BASE) return; @@ -677,39 +698,24 @@ static void __init resource_init(void) code_resource.end = __pa_symbol(&_etext) - 1; data_resource.start = __pa_symbol(&_etext); data_resource.end = __pa_symbol(&_edata) - 1; + bss_resource.start = __pa_symbol(&__bss_start); + bss_resource.end = __pa_symbol(&__bss_stop) - 1; - /* - * Request address space for all standard RAM. - */ - mips_parse_crashkernel(); - - for (i = 0; i < boot_mem_map.nr_map; i++) { + for_each_mem_range(i, &start, &end) { struct resource *res; - unsigned long start, end; - start = boot_mem_map.map[i].addr; - end = boot_mem_map.map[i].addr + boot_mem_map.map[i].size - 1; - if (start >= HIGHMEM_START) - continue; - if (end >= HIGHMEM_START) - end = HIGHMEM_START - 1; - - res = alloc_bootmem(sizeof(struct resource)); - switch (boot_mem_map.map[i].type) { - case BOOT_MEM_RAM: - case BOOT_MEM_INIT_RAM: - case BOOT_MEM_ROM_DATA: - res->name = "System RAM"; - break; - case BOOT_MEM_RESERVED: - default: - res->name = "reserved"; - } + res = memblock_alloc_or_panic(sizeof(struct resource), SMP_CACHE_BYTES); res->start = start; - res->end = end; + /* + * In memblock, end points to the first byte after the + * range while in resourses, end points to the last byte in + * the range. + */ + res->end = end - 1; + res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; + res->name = "System RAM"; - res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; request_resource(&iomem_resource, res); /* @@ -719,35 +725,75 @@ static void __init resource_init(void) */ request_resource(res, &code_resource); request_resource(res, &data_resource); + request_resource(res, &bss_resource); request_crashkernel(res); } } +#ifdef CONFIG_SMP +static void __init prefill_possible_map(void) +{ + int i, possible = num_possible_cpus(); + + if (possible > nr_cpu_ids) + possible = nr_cpu_ids; + + for (i = 0; i < possible; i++) + set_cpu_possible(i, true); + for (; i < NR_CPUS; i++) + set_cpu_possible(i, false); + + set_nr_cpu_ids(possible); +} +#else +static inline void prefill_possible_map(void) {} +#endif + +static void __init setup_rng_seed(void) +{ + char *rng_seed_hex = fw_getenv("rngseed"); + u8 rng_seed[512]; + size_t len; + + if (!rng_seed_hex) + return; + + len = min(sizeof(rng_seed), strlen(rng_seed_hex) / 2); + if (hex2bin(rng_seed, rng_seed_hex, len)) + return; + + add_bootloader_randomness(rng_seed, len); + memzero_explicit(rng_seed, len); + memzero_explicit(rng_seed_hex, len * 2); +} + void __init setup_arch(char **cmdline_p) { cpu_probe(); + mips_cm_probe(); prom_init(); + setup_early_fdc_console(); #ifdef CONFIG_EARLY_PRINTK setup_early_printk(); #endif cpu_report(); - check_bugs_early(); - -#if defined(CONFIG_VT) -#if defined(CONFIG_VGA_CONSOLE) - conswitchp = &vga_con; -#elif defined(CONFIG_DUMMY_CONSOLE) - conswitchp = &dummy_con; -#endif -#endif + if (IS_ENABLED(CONFIG_CPU_R4X00_BUGS64)) + check_bugs64_early(); arch_mem_init(cmdline_p); + dmi_setup(); resource_init(); plat_smp_setup(); + prefill_possible_map(); cpu_cache_init(); + paging_init(); + + memblock_dump_all(); + + setup_rng_seed(); } unsigned long kernelsp[NR_CPUS]; @@ -757,13 +803,37 @@ unsigned long fw_arg0, fw_arg1, fw_arg2, fw_arg3; struct dentry *mips_debugfs_dir; static int __init debugfs_mips(void) { - struct dentry *d; - - d = debugfs_create_dir("mips", NULL); - if (!d) - return -ENOMEM; - mips_debugfs_dir = d; + mips_debugfs_dir = debugfs_create_dir("mips", NULL); return 0; } arch_initcall(debugfs_mips); #endif + +#ifdef CONFIG_DMA_NONCOHERENT +static int __init setcoherentio(char *str) +{ + dma_default_coherent = true; + pr_info("Hardware DMA cache coherency (command line)\n"); + return 0; +} +early_param("coherentio", setcoherentio); + +static int __init setnocoherentio(char *str) +{ + dma_default_coherent = false; + pr_info("Software DMA cache coherency (command line)\n"); + return 0; +} +early_param("nocoherentio", setnocoherentio); +#endif + +void __init arch_cpu_finalize_init(void) +{ + unsigned int cpu = smp_processor_id(); + + cpu_data[cpu].udelay_val = loops_per_jiffy; + check_bugs32(); + + if (IS_ENABLED(CONFIG_CPU_R4X00_BUGS64)) + check_bugs64(); +} diff --git a/arch/mips/kernel/signal-common.h b/arch/mips/kernel/signal-common.h index 9c60d09e62a7..136eb20ac024 100644 --- a/arch/mips/kernel/signal-common.h +++ b/arch/mips/kernel/signal-common.h @@ -22,18 +22,25 @@ /* * Determine which stack to use.. */ -extern void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, +extern void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, size_t frame_size); /* Check and clear pending FPU exceptions in saved CSR */ extern int fpcsr_pending(unsigned int __user *fpcsr); /* Make sure we will not lose FPU ownership */ -#ifdef CONFIG_PREEMPT -#define lock_fpu_owner() preempt_disable() -#define unlock_fpu_owner() preempt_enable() -#else -#define lock_fpu_owner() pagefault_disable() -#define unlock_fpu_owner() pagefault_enable() -#endif +#define lock_fpu_owner() ({ preempt_disable(); pagefault_disable(); }) +#define unlock_fpu_owner() ({ pagefault_enable(); preempt_enable(); }) + +/* Assembly functions to move context to/from the FPU */ +extern asmlinkage int +_save_fp_context(void __user *fpregs, void __user *csr); +extern asmlinkage int +_restore_fp_context(void __user *fpregs, void __user *csr); + +extern asmlinkage int _save_msa_all_upper(void __user *buf); +extern asmlinkage int _restore_msa_all_upper(void __user *buf); + +extern int setup_sigcontext(struct pt_regs *, struct sigcontext __user *); +extern int restore_sigcontext(struct pt_regs *, struct sigcontext __user *); #endif /* __SIGNAL_COMMON_H */ diff --git a/arch/mips/kernel/signal.c b/arch/mips/kernel/signal.c index 2f285abc76d5..4a10f18a8806 100644 --- a/arch/mips/kernel/signal.c +++ b/arch/mips/kernel/signal.c @@ -6,6 +6,7 @@ * Copyright (C) 1991, 1992 Linus Torvalds * Copyright (C) 1994 - 2000 Ralf Baechle * Copyright (C) 1999, 2000 Silicon Graphics, Inc. + * Copyright (C) 2014, Imagination Technologies Ltd. */ #include <linux/cache.h> #include <linux/context_tracking.h> @@ -20,10 +21,11 @@ #include <linux/wait.h> #include <linux/ptrace.h> #include <linux/unistd.h> +#include <linux/uprobes.h> #include <linux/compiler.h> #include <linux/syscalls.h> #include <linux/uaccess.h> -#include <linux/tracehook.h> +#include <linux/resume_user_mode.h> #include <asm/abi.h> #include <asm/asm.h> @@ -33,27 +35,24 @@ #include <asm/sim.h> #include <asm/ucontext.h> #include <asm/cpu-features.h> -#include <asm/war.h> -#include <asm/vdso.h> #include <asm/dsp.h> #include <asm/inst.h> +#include <asm/msa.h> +#include <asm/syscalls.h> #include "signal-common.h" -static int (*save_fp_context)(struct sigcontext __user *sc); -static int (*restore_fp_context)(struct sigcontext __user *sc); - -extern asmlinkage int _save_fp_context(struct sigcontext __user *sc); -extern asmlinkage int _restore_fp_context(struct sigcontext __user *sc); - -extern asmlinkage int fpu_emulator_save_context(struct sigcontext __user *sc); -extern asmlinkage int fpu_emulator_restore_context(struct sigcontext __user *sc); +static int (*save_fp_context)(void __user *sc); +static int (*restore_fp_context)(void __user *sc); struct sigframe { u32 sf_ass[4]; /* argument save space for o32 */ u32 sf_pad[2]; /* Was: signal trampoline */ + + /* Matches struct ucontext from its uc_mcontext field onwards */ struct sigcontext sf_sc; sigset_t sf_mask; + unsigned long long sf_extcontext[]; }; struct rt_sigframe { @@ -63,54 +62,384 @@ struct rt_sigframe { struct ucontext rs_uc; }; +#ifdef CONFIG_MIPS_FP_SUPPORT + +/* + * Thread saved context copy to/from a signal context presumed to be on the + * user stack, and therefore accessed with appropriate macros from uaccess.h. + */ +static int copy_fp_to_sigcontext(void __user *sc) +{ + struct mips_abi *abi = current->thread.abi; + uint64_t __user *fpregs = sc + abi->off_sc_fpregs; + uint32_t __user *csr = sc + abi->off_sc_fpc_csr; + int i; + int err = 0; + int inc = test_thread_flag(TIF_32BIT_FPREGS) ? 2 : 1; + + for (i = 0; i < NUM_FPU_REGS; i += inc) { + err |= + __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 0), + &fpregs[i]); + } + err |= __put_user(current->thread.fpu.fcr31, csr); + + return err; +} + +static int copy_fp_from_sigcontext(void __user *sc) +{ + struct mips_abi *abi = current->thread.abi; + uint64_t __user *fpregs = sc + abi->off_sc_fpregs; + uint32_t __user *csr = sc + abi->off_sc_fpc_csr; + int i; + int err = 0; + int inc = test_thread_flag(TIF_32BIT_FPREGS) ? 2 : 1; + u64 fpr_val; + + for (i = 0; i < NUM_FPU_REGS; i += inc) { + err |= __get_user(fpr_val, &fpregs[i]); + set_fpr64(¤t->thread.fpu.fpr[i], 0, fpr_val); + } + err |= __get_user(current->thread.fpu.fcr31, csr); + + return err; +} + +#else /* !CONFIG_MIPS_FP_SUPPORT */ + +static int copy_fp_to_sigcontext(void __user *sc) +{ + return 0; +} + +static int copy_fp_from_sigcontext(void __user *sc) +{ + return 0; +} + +#endif /* !CONFIG_MIPS_FP_SUPPORT */ + +/* + * Wrappers for the assembly _{save,restore}_fp_context functions. + */ +static int save_hw_fp_context(void __user *sc) +{ + struct mips_abi *abi = current->thread.abi; + uint64_t __user *fpregs = sc + abi->off_sc_fpregs; + uint32_t __user *csr = sc + abi->off_sc_fpc_csr; + + return _save_fp_context(fpregs, csr); +} + +static int restore_hw_fp_context(void __user *sc) +{ + struct mips_abi *abi = current->thread.abi; + uint64_t __user *fpregs = sc + abi->off_sc_fpregs; + uint32_t __user *csr = sc + abi->off_sc_fpc_csr; + + return _restore_fp_context(fpregs, csr); +} + +/* + * Extended context handling. + */ + +static inline void __user *sc_to_extcontext(void __user *sc) +{ + struct ucontext __user *uc; + + /* + * We can just pretend the sigcontext is always embedded in a struct + * ucontext here, because the offset from sigcontext to extended + * context is the same in the struct sigframe case. + */ + uc = container_of(sc, struct ucontext, uc_mcontext); + return &uc->uc_extcontext; +} + +#ifdef CONFIG_CPU_HAS_MSA + +static int save_msa_extcontext(void __user *buf) +{ + struct msa_extcontext __user *msa = buf; + uint64_t val; + int i, err; + + if (!thread_msa_context_live()) + return 0; + + /* + * Ensure that we can't lose the live MSA context between checking + * for it & writing it to memory. + */ + preempt_disable(); + + if (is_msa_enabled()) { + /* + * There are no EVA versions of the vector register load/store + * instructions, so MSA context has to be saved to kernel memory + * and then copied to user memory. The save to kernel memory + * should already have been done when handling scalar FP + * context. + */ + BUG_ON(IS_ENABLED(CONFIG_EVA)); + + err = __put_user(read_msa_csr(), &msa->csr); + err |= _save_msa_all_upper(&msa->wr); + + preempt_enable(); + } else { + preempt_enable(); + + err = __put_user(current->thread.fpu.msacsr, &msa->csr); + + for (i = 0; i < NUM_FPU_REGS; i++) { + val = get_fpr64(¤t->thread.fpu.fpr[i], 1); + err |= __put_user(val, &msa->wr[i]); + } + } + + err |= __put_user(MSA_EXTCONTEXT_MAGIC, &msa->ext.magic); + err |= __put_user(sizeof(*msa), &msa->ext.size); + + return err ? -EFAULT : sizeof(*msa); +} + +static int restore_msa_extcontext(void __user *buf, unsigned int size) +{ + struct msa_extcontext __user *msa = buf; + unsigned long long val; + unsigned int csr; + int i, err; + + if (size != sizeof(*msa)) + return -EINVAL; + + err = get_user(csr, &msa->csr); + if (err) + return err; + + preempt_disable(); + + if (is_msa_enabled()) { + /* + * There are no EVA versions of the vector register load/store + * instructions, so MSA context has to be copied to kernel + * memory and later loaded to registers. The same is true of + * scalar FP context, so FPU & MSA should have already been + * disabled whilst handling scalar FP context. + */ + BUG_ON(IS_ENABLED(CONFIG_EVA)); + + write_msa_csr(csr); + err |= _restore_msa_all_upper(&msa->wr); + preempt_enable(); + } else { + preempt_enable(); + + current->thread.fpu.msacsr = csr; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= __get_user(val, &msa->wr[i]); + set_fpr64(¤t->thread.fpu.fpr[i], 1, val); + } + } + + return err; +} + +#else /* !CONFIG_CPU_HAS_MSA */ + +static int save_msa_extcontext(void __user *buf) +{ + return 0; +} + +static int restore_msa_extcontext(void __user *buf, unsigned int size) +{ + return SIGSYS; +} + +#endif /* !CONFIG_CPU_HAS_MSA */ + +static int save_extcontext(void __user *buf) +{ + int sz; + + sz = save_msa_extcontext(buf); + if (sz < 0) + return sz; + buf += sz; + + /* If no context was saved then trivially return */ + if (!sz) + return 0; + + /* Write the end marker */ + if (__put_user(END_EXTCONTEXT_MAGIC, (u32 *)buf)) + return -EFAULT; + + sz += sizeof(((struct extcontext *)NULL)->magic); + return sz; +} + +static int restore_extcontext(void __user *buf) +{ + struct extcontext ext; + int err; + + while (1) { + err = __get_user(ext.magic, (unsigned int *)buf); + if (err) + return err; + + if (ext.magic == END_EXTCONTEXT_MAGIC) + return 0; + + err = __get_user(ext.size, (unsigned int *)(buf + + offsetof(struct extcontext, size))); + if (err) + return err; + + switch (ext.magic) { + case MSA_EXTCONTEXT_MAGIC: + err = restore_msa_extcontext(buf, ext.size); + break; + + default: + err = -EINVAL; + break; + } + + if (err) + return err; + + buf += ext.size; + } +} + /* * Helper routines */ -static int protected_save_fp_context(struct sigcontext __user *sc) +int protected_save_fp_context(void __user *sc) { + struct mips_abi *abi = current->thread.abi; + uint64_t __user *fpregs = sc + abi->off_sc_fpregs; + uint32_t __user *csr = sc + abi->off_sc_fpc_csr; + uint32_t __user *used_math = sc + abi->off_sc_used_math; + unsigned int used, ext_sz; int err; + + used = used_math() ? USED_FP : 0; + if (!used) + goto fp_done; + + if (!test_thread_flag(TIF_32BIT_FPREGS)) + used |= USED_FR1; + if (test_thread_flag(TIF_HYBRID_FPREGS)) + used |= USED_HYBRID_FPRS; + + /* + * EVA does not have userland equivalents of ldc1 or sdc1, so + * save to the kernel FP context & copy that to userland below. + */ + if (IS_ENABLED(CONFIG_EVA)) + lose_fpu(1); + while (1) { lock_fpu_owner(); - own_fpu_inatomic(1); - err = save_fp_context(sc); /* this might fail */ - unlock_fpu_owner(); + if (is_fpu_owner()) { + err = save_fp_context(sc); + unlock_fpu_owner(); + } else { + unlock_fpu_owner(); + err = copy_fp_to_sigcontext(sc); + } if (likely(!err)) break; /* touch the sigcontext and try again */ - err = __put_user(0, &sc->sc_fpregs[0]) | - __put_user(0, &sc->sc_fpregs[31]) | - __put_user(0, &sc->sc_fpc_csr); + err = __put_user(0, &fpregs[0]) | + __put_user(0, &fpregs[31]) | + __put_user(0, csr); if (err) - break; /* really bad sigcontext */ + return err; /* really bad sigcontext */ } - return err; + +fp_done: + ext_sz = err = save_extcontext(sc_to_extcontext(sc)); + if (err < 0) + return err; + used |= ext_sz ? USED_EXTCONTEXT : 0; + + return __put_user(used, used_math); } -static int protected_restore_fp_context(struct sigcontext __user *sc) +int protected_restore_fp_context(void __user *sc) { - int err, tmp __maybe_unused; + struct mips_abi *abi = current->thread.abi; + uint64_t __user *fpregs = sc + abi->off_sc_fpregs; + uint32_t __user *csr = sc + abi->off_sc_fpc_csr; + uint32_t __user *used_math = sc + abi->off_sc_used_math; + unsigned int used; + int err, sig = 0, tmp __maybe_unused; + + err = __get_user(used, used_math); + conditional_used_math(used & USED_FP); + + /* + * The signal handler may have used FPU; give it up if the program + * doesn't want it following sigreturn. + */ + if (err || !(used & USED_FP)) + lose_fpu(0); + if (err) + return err; + if (!(used & USED_FP)) + goto fp_done; + + err = sig = fpcsr_pending(csr); + if (err < 0) + return err; + + /* + * EVA does not have userland equivalents of ldc1 or sdc1, so we + * disable the FPU here such that the code below simply copies to + * the kernel FP context. + */ + if (IS_ENABLED(CONFIG_EVA)) + lose_fpu(0); + while (1) { lock_fpu_owner(); - own_fpu_inatomic(0); - err = restore_fp_context(sc); /* this might fail */ - unlock_fpu_owner(); + if (is_fpu_owner()) { + err = restore_fp_context(sc); + unlock_fpu_owner(); + } else { + unlock_fpu_owner(); + err = copy_fp_from_sigcontext(sc); + } if (likely(!err)) break; /* touch the sigcontext and try again */ - err = __get_user(tmp, &sc->sc_fpregs[0]) | - __get_user(tmp, &sc->sc_fpregs[31]) | - __get_user(tmp, &sc->sc_fpc_csr); + err = __get_user(tmp, &fpregs[0]) | + __get_user(tmp, &fpregs[31]) | + __get_user(tmp, csr); if (err) break; /* really bad sigcontext */ } - return err; + +fp_done: + if (!err && (used & USED_EXTCONTEXT)) + err = restore_extcontext(sc_to_extcontext(sc)); + + return err ?: sig; } int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) { int err = 0; int i; - unsigned int used_math; err |= __put_user(regs->cp0_epc, &sc->sc_pc); @@ -133,19 +462,38 @@ int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) err |= __put_user(rddsp(DSP_MASK), &sc->sc_dsp); } - used_math = !!used_math(); - err |= __put_user(used_math, &sc->sc_used_math); - if (used_math) { - /* - * Save FPU state to signal context. Signal handler - * will "inherit" current FPU state. - */ - err |= protected_save_fp_context(sc); - } + /* + * Save FPU state to signal context. Signal handler + * will "inherit" current FPU state. + */ + err |= protected_save_fp_context(sc); + return err; } +static size_t extcontext_max_size(void) +{ + size_t sz = 0; + + /* + * The assumption here is that between this point & the point at which + * the extended context is saved the size of the context should only + * ever be able to shrink (if the task is preempted), but never grow. + * That is, what this function returns is an upper bound on the size of + * the extended context for the current task at the current time. + */ + + if (thread_msa_context_live()) + sz += sizeof(struct msa_extcontext); + + /* If any context is saved then we'll append the end marker */ + if (sz) + sz += sizeof(((struct extcontext *)NULL)->magic); + + return sz; +} + int fpcsr_pending(unsigned int __user *fpcsr) { int err, sig = 0; @@ -165,27 +513,14 @@ int fpcsr_pending(unsigned int __user *fpcsr) return err ?: sig; } -static int -check_and_restore_fp_context(struct sigcontext __user *sc) -{ - int err, sig; - - err = sig = fpcsr_pending(&sc->sc_fpc_csr); - if (err > 0) - err = 0; - err |= protected_restore_fp_context(sc); - return err ?: sig; -} - int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) { - unsigned int used_math; unsigned long treg; int err = 0; int i; /* Always make any pending restarted system calls return -EINTR */ - current_thread_info()->restart_block.fn = do_no_restart_syscall; + current->restart_block.fn = do_no_restart_syscall; err |= __get_user(regs->cp0_epc, &sc->sc_pc); @@ -207,41 +542,43 @@ int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) for (i = 1; i < 32; i++) err |= __get_user(regs->regs[i], &sc->sc_regs[i]); - err |= __get_user(used_math, &sc->sc_used_math); - conditional_used_math(used_math); - - if (used_math) { - /* restore fpu context if we have used it before */ - if (!err) - err = check_and_restore_fp_context(sc); - } else { - /* signal handler may have used FPU. Give it up. */ - lose_fpu(0); - } - - return err; + return err ?: protected_restore_fp_context(sc); } -void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, +#ifdef CONFIG_WAR_ICACHE_REFILLS +#define SIGMASK ~(cpu_icache_line_size()-1) +#else +#define SIGMASK ALMASK +#endif + +void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, size_t frame_size) { unsigned long sp; + /* Leave space for potential extended context */ + frame_size += extcontext_max_size(); + /* Default to using normal stack */ sp = regs->regs[29]; /* - * FPU emulator may have it's own trampoline active just + * If we are on the alternate signal stack and would overflow it, don't. + * Return an always-bogus address instead so we will die with SIGSEGV. + */ + if (on_sig_stack(sp) && !likely(on_sig_stack(sp - frame_size))) + return (void __user __force *)(-1UL); + + /* + * FPU emulator may have its own trampoline active just * above the user stack, 16-bytes before the next lowest * 16 byte boundary. Try to avoid trashing it. */ sp -= 32; - /* This is the X/Open sanctioned signal stack switching. */ - if ((ka->sa.sa_flags & SA_ONSTACK) && (sas_ss_flags (sp) == 0)) - sp = current->sas_ss_sp + current->sas_ss_size; + sp = sigsp(sp, ksig); - return (void __user *)((sp - frame_size) & (ICACHE_REFILLS_WORKAROUND_WAR ? ~(cpu_icache_line_size()-1) : ALMASK)); + return (void __user *)((sp - frame_size) & SIGMASK); } /* @@ -266,7 +603,7 @@ SYSCALL_DEFINE3(sigaction, int, sig, const struct sigaction __user *, act, if (act) { old_sigset_t mask; - if (!access_ok(VERIFY_READ, act, sizeof(*act))) + if (!access_ok(act, sizeof(*act))) return -EFAULT; err |= __get_user(new_ka.sa.sa_handler, &act->sa_handler); err |= __get_user(new_ka.sa.sa_flags, &act->sa_flags); @@ -280,7 +617,7 @@ SYSCALL_DEFINE3(sigaction, int, sig, const struct sigaction __user *, act, ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); if (!ret && oact) { - if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact))) + if (!access_ok(oact, sizeof(*oact))) return -EFAULT; err |= __put_user(old_ka.sa.sa_flags, &oact->sa_flags); err |= __put_user(old_ka.sa.sa_handler, &oact->sa_handler); @@ -297,25 +634,27 @@ SYSCALL_DEFINE3(sigaction, int, sig, const struct sigaction __user *, act, #endif #ifdef CONFIG_TRAD_SIGNALS -asmlinkage void sys_sigreturn(nabi_no_regargs struct pt_regs regs) +asmlinkage void sys_sigreturn(void) { struct sigframe __user *frame; + struct pt_regs *regs; sigset_t blocked; int sig; - frame = (struct sigframe __user *) regs.regs[29]; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + regs = current_pt_regs(); + frame = (struct sigframe __user *)regs->regs[29]; + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&blocked, &frame->sf_mask, sizeof(blocked))) goto badframe; set_current_blocked(&blocked); - sig = restore_sigcontext(®s, &frame->sf_sc); + sig = restore_sigcontext(regs, &frame->sf_sc); if (sig < 0) goto badframe; else if (sig) - force_sig(sig, current); + force_sig(sig); /* * Don't let your children do this ... @@ -323,34 +662,36 @@ asmlinkage void sys_sigreturn(nabi_no_regargs struct pt_regs regs) __asm__ __volatile__( "move\t$29, %0\n\t" "j\tsyscall_exit" - :/* no outputs */ - :"r" (®s)); + : /* no outputs */ + : "r" (regs)); /* Unreached */ badframe: - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); } #endif /* CONFIG_TRAD_SIGNALS */ -asmlinkage void sys_rt_sigreturn(nabi_no_regargs struct pt_regs regs) +asmlinkage void sys_rt_sigreturn(void) { struct rt_sigframe __user *frame; + struct pt_regs *regs; sigset_t set; int sig; - frame = (struct rt_sigframe __user *) regs.regs[29]; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + regs = current_pt_regs(); + frame = (struct rt_sigframe __user *)regs->regs[29]; + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_from_user(&set, &frame->rs_uc.uc_sigmask, sizeof(set))) goto badframe; set_current_blocked(&set); - sig = restore_sigcontext(®s, &frame->rs_uc.uc_mcontext); + sig = restore_sigcontext(regs, &frame->rs_uc.uc_mcontext); if (sig < 0) goto badframe; else if (sig) - force_sig(sig, current); + force_sig(sig); if (restore_altstack(&frame->rs_uc.uc_stack)) goto badframe; @@ -361,29 +702,29 @@ asmlinkage void sys_rt_sigreturn(nabi_no_regargs struct pt_regs regs) __asm__ __volatile__( "move\t$29, %0\n\t" "j\tsyscall_exit" - :/* no outputs */ - :"r" (®s)); + : /* no outputs */ + : "r" (regs)); /* Unreached */ badframe: - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); } #ifdef CONFIG_TRAD_SIGNALS -static int setup_frame(void *sig_return, struct k_sigaction *ka, - struct pt_regs *regs, int signr, sigset_t *set) +static int setup_frame(void *sig_return, struct ksignal *ksig, + struct pt_regs *regs, sigset_t *set) { struct sigframe __user *frame; int err = 0; - frame = get_sigframe(ka, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame))) - goto give_sigsegv; + frame = get_sigframe(ksig, regs, sizeof(*frame)); + if (!access_ok(frame, sizeof (*frame))) + return -EFAULT; err |= setup_sigcontext(regs, &frame->sf_sc); err |= __copy_to_user(&frame->sf_mask, set, sizeof(*set)); if (err) - goto give_sigsegv; + return -EFAULT; /* * Arguments to signal handler: @@ -395,47 +736,44 @@ static int setup_frame(void *sig_return, struct k_sigaction *ka, * $25 and c0_epc point to the signal handler, $29 points to the * struct sigframe. */ - regs->regs[ 4] = signr; + regs->regs[ 4] = ksig->sig; regs->regs[ 5] = 0; regs->regs[ 6] = (unsigned long) &frame->sf_sc; regs->regs[29] = (unsigned long) frame; regs->regs[31] = (unsigned long) sig_return; - regs->cp0_epc = regs->regs[25] = (unsigned long) ka->sa.sa_handler; + regs->cp0_epc = regs->regs[25] = (unsigned long) ksig->ka.sa.sa_handler; DEBUGP("SIG deliver (%s:%d): sp=0x%p pc=0x%lx ra=0x%lx\n", current->comm, current->pid, frame, regs->cp0_epc, regs->regs[31]); return 0; - -give_sigsegv: - force_sigsegv(signr, current); - return -EFAULT; } #endif -static int setup_rt_frame(void *sig_return, struct k_sigaction *ka, - struct pt_regs *regs, int signr, sigset_t *set, - siginfo_t *info) +static int setup_rt_frame(void *sig_return, struct ksignal *ksig, + struct pt_regs *regs, sigset_t *set) { struct rt_sigframe __user *frame; - int err = 0; - frame = get_sigframe(ka, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame))) - goto give_sigsegv; + frame = get_sigframe(ksig, regs, sizeof(*frame)); + if (!access_ok(frame, sizeof (*frame))) + return -EFAULT; /* Create siginfo. */ - err |= copy_siginfo_to_user(&frame->rs_info, info); + if (copy_siginfo_to_user(&frame->rs_info, &ksig->info)) + return -EFAULT; /* Create the ucontext. */ - err |= __put_user(0, &frame->rs_uc.uc_flags); - err |= __put_user(NULL, &frame->rs_uc.uc_link); - err |= __save_altstack(&frame->rs_uc.uc_stack, regs->regs[29]); - err |= setup_sigcontext(regs, &frame->rs_uc.uc_mcontext); - err |= __copy_to_user(&frame->rs_uc.uc_sigmask, set, sizeof(*set)); - - if (err) - goto give_sigsegv; + if (__put_user(0, &frame->rs_uc.uc_flags)) + return -EFAULT; + if (__put_user(NULL, &frame->rs_uc.uc_link)) + return -EFAULT; + if (__save_altstack(&frame->rs_uc.uc_stack, regs->regs[29])) + return -EFAULT; + if (setup_sigcontext(regs, &frame->rs_uc.uc_mcontext)) + return -EFAULT; + if (__copy_to_user(&frame->rs_uc.uc_sigmask, set, sizeof(*set))) + return -EFAULT; /* * Arguments to signal handler: @@ -447,50 +785,48 @@ static int setup_rt_frame(void *sig_return, struct k_sigaction *ka, * $25 and c0_epc point to the signal handler, $29 points to * the struct rt_sigframe. */ - regs->regs[ 4] = signr; + regs->regs[ 4] = ksig->sig; regs->regs[ 5] = (unsigned long) &frame->rs_info; regs->regs[ 6] = (unsigned long) &frame->rs_uc; regs->regs[29] = (unsigned long) frame; regs->regs[31] = (unsigned long) sig_return; - regs->cp0_epc = regs->regs[25] = (unsigned long) ka->sa.sa_handler; + regs->cp0_epc = regs->regs[25] = (unsigned long) ksig->ka.sa.sa_handler; DEBUGP("SIG deliver (%s:%d): sp=0x%p pc=0x%lx ra=0x%lx\n", current->comm, current->pid, frame, regs->cp0_epc, regs->regs[31]); return 0; - -give_sigsegv: - force_sigsegv(signr, current); - return -EFAULT; } struct mips_abi mips_abi = { #ifdef CONFIG_TRAD_SIGNALS .setup_frame = setup_frame, - .signal_return_offset = offsetof(struct mips_vdso, signal_trampoline), #endif .setup_rt_frame = setup_rt_frame, - .rt_signal_return_offset = - offsetof(struct mips_vdso, rt_signal_trampoline), - .restart = __NR_restart_syscall + .restart = __NR_restart_syscall, + + .off_sc_fpregs = offsetof(struct sigcontext, sc_fpregs), + .off_sc_fpc_csr = offsetof(struct sigcontext, sc_fpc_csr), + .off_sc_used_math = offsetof(struct sigcontext, sc_used_math), + + .vdso = &vdso_image, }; -static void handle_signal(unsigned long sig, siginfo_t *info, - struct k_sigaction *ka, struct pt_regs *regs) +static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) { sigset_t *oldset = sigmask_to_save(); int ret; struct mips_abi *abi = current->thread.abi; -#ifdef CONFIG_CPU_MICROMIPS - void *vdso; - unsigned int tmp = (unsigned int)current->mm->context.vdso; - - set_isa16_mode(tmp); - vdso = (void *)tmp; -#else void *vdso = current->mm->context.vdso; -#endif + + /* + * If we were emulating a delay slot instruction, exit that frame such + * that addresses in the sigframe are as expected for userland and we + * don't have a problem if we reuse the thread's frame for an + * instruction within the signal handler. + */ + dsemul_thread_rollback(regs); if (regs->regs[0]) { switch(regs->regs[2]) { @@ -499,11 +835,11 @@ static void handle_signal(unsigned long sig, siginfo_t *info, regs->regs[2] = EINTR; break; case ERESTARTSYS: - if (!(ka->sa.sa_flags & SA_RESTART)) { + if (!(ksig->ka.sa.sa_flags & SA_RESTART)) { regs->regs[2] = EINTR; break; } - /* fallthrough */ + fallthrough; case ERESTARTNOINTR: regs->regs[7] = regs->regs[26]; regs->regs[2] = regs->regs[0]; @@ -513,29 +849,25 @@ static void handle_signal(unsigned long sig, siginfo_t *info, regs->regs[0] = 0; /* Don't deal with this again. */ } - if (sig_uses_siginfo(ka)) - ret = abi->setup_rt_frame(vdso + abi->rt_signal_return_offset, - ka, regs, sig, oldset, info); - else - ret = abi->setup_frame(vdso + abi->signal_return_offset, - ka, regs, sig, oldset); + rseq_signal_deliver(ksig, regs); - if (ret) - return; + if (sig_uses_siginfo(&ksig->ka, abi)) + ret = abi->setup_rt_frame(vdso + abi->vdso->off_rt_sigreturn, + ksig, regs, oldset); + else + ret = abi->setup_frame(vdso + abi->vdso->off_sigreturn, + ksig, regs, oldset); - signal_delivered(sig, info, ka, regs, 0); + signal_setup_done(ret, ksig, 0); } static void do_signal(struct pt_regs *regs) { - struct k_sigaction ka; - siginfo_t info; - int signr; + struct ksignal ksig; - signr = get_signal_to_deliver(&info, &ka, regs, NULL); - if (signr > 0) { + if (get_signal(&ksig)) { /* Whee! Actually deliver the signal. */ - handle_signal(signr, &info, &ka, regs); + handle_signal(&ksig, regs); return; } @@ -576,49 +908,60 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, void *unused, user_exit(); + if (thread_info_flags & _TIF_UPROBE) + uprobe_notify_resume(regs); + /* deal with pending signal delivery */ - if (thread_info_flags & _TIF_SIGPENDING) + if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) do_signal(regs); - if (thread_info_flags & _TIF_NOTIFY_RESUME) { - clear_thread_flag(TIF_NOTIFY_RESUME); - tracehook_notify_resume(regs); - } + if (thread_info_flags & _TIF_NOTIFY_RESUME) + resume_user_mode_work(regs); user_enter(); } -#ifdef CONFIG_SMP -static int smp_save_fp_context(struct sigcontext __user *sc) +#if defined(CONFIG_SMP) && defined(CONFIG_MIPS_FP_SUPPORT) +static int smp_save_fp_context(void __user *sc) { return raw_cpu_has_fpu - ? _save_fp_context(sc) - : fpu_emulator_save_context(sc); + ? save_hw_fp_context(sc) + : copy_fp_to_sigcontext(sc); } -static int smp_restore_fp_context(struct sigcontext __user *sc) +static int smp_restore_fp_context(void __user *sc) { return raw_cpu_has_fpu - ? _restore_fp_context(sc) - : fpu_emulator_restore_context(sc); + ? restore_hw_fp_context(sc) + : copy_fp_from_sigcontext(sc); } #endif static int signal_setup(void) { -#ifdef CONFIG_SMP + /* + * The offset from sigcontext to extended context should be the same + * regardless of the type of signal, such that userland can always know + * where to look if it wishes to find the extended context structures. + */ + BUILD_BUG_ON((offsetof(struct sigframe, sf_extcontext) - + offsetof(struct sigframe, sf_sc)) != + (offsetof(struct rt_sigframe, rs_uc.uc_extcontext) - + offsetof(struct rt_sigframe, rs_uc.uc_mcontext))); + +#if defined(CONFIG_SMP) && defined(CONFIG_MIPS_FP_SUPPORT) /* For now just do the cpu_has_fpu check when the functions are invoked */ save_fp_context = smp_save_fp_context; restore_fp_context = smp_restore_fp_context; #else if (cpu_has_fpu) { - save_fp_context = _save_fp_context; - restore_fp_context = _restore_fp_context; + save_fp_context = save_hw_fp_context; + restore_fp_context = restore_hw_fp_context; } else { - save_fp_context = fpu_emulator_save_context; - restore_fp_context = fpu_emulator_restore_context; + save_fp_context = copy_fp_to_sigcontext; + restore_fp_context = copy_fp_from_sigcontext; } -#endif +#endif /* CONFIG_SMP */ return 0; } diff --git a/arch/mips/kernel/signal32.c b/arch/mips/kernel/signal32.c index 57de8b751627..73081d4ee8c1 100644 --- a/arch/mips/kernel/signal32.c +++ b/arch/mips/kernel/signal32.c @@ -6,263 +6,27 @@ * Copyright (C) 1991, 1992 Linus Torvalds * Copyright (C) 1994 - 2000, 2006 Ralf Baechle * Copyright (C) 1999, 2000 Silicon Graphics, Inc. + * Copyright (C) 2016, Imagination Technologies Ltd. */ -#include <linux/cache.h> #include <linux/compat.h> -#include <linux/sched.h> -#include <linux/mm.h> -#include <linux/smp.h> +#include <linux/compiler.h> +#include <linux/errno.h> #include <linux/kernel.h> #include <linux/signal.h> #include <linux/syscalls.h> -#include <linux/errno.h> -#include <linux/wait.h> -#include <linux/ptrace.h> -#include <linux/suspend.h> -#include <linux/compiler.h> -#include <linux/uaccess.h> -#include <asm/abi.h> -#include <asm/asm.h> #include <asm/compat-signal.h> -#include <linux/bitops.h> -#include <asm/cacheflush.h> -#include <asm/sim.h> -#include <asm/ucontext.h> -#include <asm/fpu.h> -#include <asm/war.h> -#include <asm/vdso.h> -#include <asm/dsp.h> +#include <linux/uaccess.h> +#include <asm/unistd.h> +#include <asm/syscalls.h> #include "signal-common.h" -static int (*save_fp_context32)(struct sigcontext32 __user *sc); -static int (*restore_fp_context32)(struct sigcontext32 __user *sc); - -extern asmlinkage int _save_fp_context32(struct sigcontext32 __user *sc); -extern asmlinkage int _restore_fp_context32(struct sigcontext32 __user *sc); - -extern asmlinkage int fpu_emulator_save_context32(struct sigcontext32 __user *sc); -extern asmlinkage int fpu_emulator_restore_context32(struct sigcontext32 __user *sc); - -/* - * Including <asm/unistd.h> would give use the 64-bit syscall numbers ... - */ -#define __NR_O32_restart_syscall 4253 - /* 32-bit compatibility types */ typedef unsigned int __sighandler32_t; typedef void (*vfptr_t)(void); -struct ucontext32 { - u32 uc_flags; - s32 uc_link; - compat_stack_t uc_stack; - struct sigcontext32 uc_mcontext; - compat_sigset_t uc_sigmask; /* mask last for extensibility */ -}; - -struct sigframe32 { - u32 sf_ass[4]; /* argument save space for o32 */ - u32 sf_pad[2]; /* Was: signal trampoline */ - struct sigcontext32 sf_sc; - compat_sigset_t sf_mask; -}; - -struct rt_sigframe32 { - u32 rs_ass[4]; /* argument save space for o32 */ - u32 rs_pad[2]; /* Was: signal trampoline */ - compat_siginfo_t rs_info; - struct ucontext32 rs_uc; -}; - -/* - * sigcontext handlers - */ -static int protected_save_fp_context32(struct sigcontext32 __user *sc) -{ - int err; - while (1) { - lock_fpu_owner(); - own_fpu_inatomic(1); - err = save_fp_context32(sc); /* this might fail */ - unlock_fpu_owner(); - if (likely(!err)) - break; - /* touch the sigcontext and try again */ - err = __put_user(0, &sc->sc_fpregs[0]) | - __put_user(0, &sc->sc_fpregs[31]) | - __put_user(0, &sc->sc_fpc_csr); - if (err) - break; /* really bad sigcontext */ - } - return err; -} - -static int protected_restore_fp_context32(struct sigcontext32 __user *sc) -{ - int err, tmp __maybe_unused; - while (1) { - lock_fpu_owner(); - own_fpu_inatomic(0); - err = restore_fp_context32(sc); /* this might fail */ - unlock_fpu_owner(); - if (likely(!err)) - break; - /* touch the sigcontext and try again */ - err = __get_user(tmp, &sc->sc_fpregs[0]) | - __get_user(tmp, &sc->sc_fpregs[31]) | - __get_user(tmp, &sc->sc_fpc_csr); - if (err) - break; /* really bad sigcontext */ - } - return err; -} - -static int setup_sigcontext32(struct pt_regs *regs, - struct sigcontext32 __user *sc) -{ - int err = 0; - int i; - u32 used_math; - - err |= __put_user(regs->cp0_epc, &sc->sc_pc); - - err |= __put_user(0, &sc->sc_regs[0]); - for (i = 1; i < 32; i++) - err |= __put_user(regs->regs[i], &sc->sc_regs[i]); - - err |= __put_user(regs->hi, &sc->sc_mdhi); - err |= __put_user(regs->lo, &sc->sc_mdlo); - if (cpu_has_dsp) { - err |= __put_user(rddsp(DSP_MASK), &sc->sc_dsp); - err |= __put_user(mfhi1(), &sc->sc_hi1); - err |= __put_user(mflo1(), &sc->sc_lo1); - err |= __put_user(mfhi2(), &sc->sc_hi2); - err |= __put_user(mflo2(), &sc->sc_lo2); - err |= __put_user(mfhi3(), &sc->sc_hi3); - err |= __put_user(mflo3(), &sc->sc_lo3); - } - - used_math = !!used_math(); - err |= __put_user(used_math, &sc->sc_used_math); - - if (used_math) { - /* - * Save FPU state to signal context. Signal handler - * will "inherit" current FPU state. - */ - err |= protected_save_fp_context32(sc); - } - return err; -} - -static int -check_and_restore_fp_context32(struct sigcontext32 __user *sc) -{ - int err, sig; - - err = sig = fpcsr_pending(&sc->sc_fpc_csr); - if (err > 0) - err = 0; - err |= protected_restore_fp_context32(sc); - return err ?: sig; -} - -static int restore_sigcontext32(struct pt_regs *regs, - struct sigcontext32 __user *sc) -{ - u32 used_math; - int err = 0; - s32 treg; - int i; - - /* Always make any pending restarted system calls return -EINTR */ - current_thread_info()->restart_block.fn = do_no_restart_syscall; - - err |= __get_user(regs->cp0_epc, &sc->sc_pc); - err |= __get_user(regs->hi, &sc->sc_mdhi); - err |= __get_user(regs->lo, &sc->sc_mdlo); - if (cpu_has_dsp) { - err |= __get_user(treg, &sc->sc_hi1); mthi1(treg); - err |= __get_user(treg, &sc->sc_lo1); mtlo1(treg); - err |= __get_user(treg, &sc->sc_hi2); mthi2(treg); - err |= __get_user(treg, &sc->sc_lo2); mtlo2(treg); - err |= __get_user(treg, &sc->sc_hi3); mthi3(treg); - err |= __get_user(treg, &sc->sc_lo3); mtlo3(treg); - err |= __get_user(treg, &sc->sc_dsp); wrdsp(treg, DSP_MASK); - } - - for (i = 1; i < 32; i++) - err |= __get_user(regs->regs[i], &sc->sc_regs[i]); - - err |= __get_user(used_math, &sc->sc_used_math); - conditional_used_math(used_math); - - if (used_math) { - /* restore fpu context if we have used it before */ - if (!err) - err = check_and_restore_fp_context32(sc); - } else { - /* signal handler may have used FPU. Give it up. */ - lose_fpu(0); - } - - return err; -} - -/* - * - */ -extern void __put_sigset_unknown_nsig(void); -extern void __get_sigset_unknown_nsig(void); - -static inline int put_sigset(const sigset_t *kbuf, compat_sigset_t __user *ubuf) -{ - int err = 0; - - if (!access_ok(VERIFY_WRITE, ubuf, sizeof(*ubuf))) - return -EFAULT; - - switch (_NSIG_WORDS) { - default: - __put_sigset_unknown_nsig(); - case 2: - err |= __put_user(kbuf->sig[1] >> 32, &ubuf->sig[3]); - err |= __put_user(kbuf->sig[1] & 0xffffffff, &ubuf->sig[2]); - case 1: - err |= __put_user(kbuf->sig[0] >> 32, &ubuf->sig[1]); - err |= __put_user(kbuf->sig[0] & 0xffffffff, &ubuf->sig[0]); - } - - return err; -} - -static inline int get_sigset(sigset_t *kbuf, const compat_sigset_t __user *ubuf) -{ - int err = 0; - unsigned long sig[4]; - - if (!access_ok(VERIFY_READ, ubuf, sizeof(*ubuf))) - return -EFAULT; - - switch (_NSIG_WORDS) { - default: - __get_sigset_unknown_nsig(); - case 2: - err |= __get_user(sig[3], &ubuf->sig[3]); - err |= __get_user(sig[2], &ubuf->sig[2]); - kbuf->sig[1] = sig[2] | (sig[3] << 32); - case 1: - err |= __get_user(sig[1], &ubuf->sig[1]); - err |= __get_user(sig[0], &ubuf->sig[0]); - kbuf->sig[0] = sig[0] | (sig[1] << 32); - } - - return err; -} - /* * Atomically swap in the new signal mask, and wait for a signal. */ @@ -283,7 +47,7 @@ SYSCALL_DEFINE3(32_sigaction, long, sig, const struct compat_sigaction __user *, old_sigset_t mask; s32 handler; - if (!access_ok(VERIFY_READ, act, sizeof(*act))) + if (!access_ok(act, sizeof(*act))) return -EFAULT; err |= __get_user(handler, &act->sa_handler); new_ka.sa.sa_handler = (void __user *)(s64)handler; @@ -298,7 +62,7 @@ SYSCALL_DEFINE3(32_sigaction, long, sig, const struct compat_sigaction __user *, ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); if (!ret && oact) { - if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact))) + if (!access_ok(oact, sizeof(*oact))) return -EFAULT; err |= __put_user(old_ka.sa.sa_flags, &oact->sa_flags); err |= __put_user((u32)(u64)old_ka.sa.sa_handler, @@ -313,262 +77,3 @@ SYSCALL_DEFINE3(32_sigaction, long, sig, const struct compat_sigaction __user *, return ret; } - -int copy_siginfo_to_user32(compat_siginfo_t __user *to, siginfo_t *from) -{ - int err; - - if (!access_ok (VERIFY_WRITE, to, sizeof(compat_siginfo_t))) - return -EFAULT; - - /* If you change siginfo_t structure, please be sure - this code is fixed accordingly. - It should never copy any pad contained in the structure - to avoid security leaks, but must copy the generic - 3 ints plus the relevant union member. - This routine must convert siginfo from 64bit to 32bit as well - at the same time. */ - err = __put_user(from->si_signo, &to->si_signo); - err |= __put_user(from->si_errno, &to->si_errno); - err |= __put_user((short)from->si_code, &to->si_code); - if (from->si_code < 0) - err |= __copy_to_user(&to->_sifields._pad, &from->_sifields._pad, SI_PAD_SIZE); - else { - switch (from->si_code >> 16) { - case __SI_TIMER >> 16: - err |= __put_user(from->si_tid, &to->si_tid); - err |= __put_user(from->si_overrun, &to->si_overrun); - err |= __put_user(from->si_int, &to->si_int); - break; - case __SI_CHLD >> 16: - err |= __put_user(from->si_utime, &to->si_utime); - err |= __put_user(from->si_stime, &to->si_stime); - err |= __put_user(from->si_status, &to->si_status); - default: - err |= __put_user(from->si_pid, &to->si_pid); - err |= __put_user(from->si_uid, &to->si_uid); - break; - case __SI_FAULT >> 16: - err |= __put_user((unsigned long)from->si_addr, &to->si_addr); - break; - case __SI_POLL >> 16: - err |= __put_user(from->si_band, &to->si_band); - err |= __put_user(from->si_fd, &to->si_fd); - break; - case __SI_RT >> 16: /* This is not generated by the kernel as of now. */ - case __SI_MESGQ >> 16: - err |= __put_user(from->si_pid, &to->si_pid); - err |= __put_user(from->si_uid, &to->si_uid); - err |= __put_user(from->si_int, &to->si_int); - break; - } - } - return err; -} - -int copy_siginfo_from_user32(siginfo_t *to, compat_siginfo_t __user *from) -{ - memset(to, 0, sizeof *to); - - if (copy_from_user(to, from, 3*sizeof(int)) || - copy_from_user(to->_sifields._pad, - from->_sifields._pad, SI_PAD_SIZE32)) - return -EFAULT; - - return 0; -} - -asmlinkage void sys32_sigreturn(nabi_no_regargs struct pt_regs regs) -{ - struct sigframe32 __user *frame; - sigset_t blocked; - int sig; - - frame = (struct sigframe32 __user *) regs.regs[29]; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) - goto badframe; - if (__copy_conv_sigset_from_user(&blocked, &frame->sf_mask)) - goto badframe; - - set_current_blocked(&blocked); - - sig = restore_sigcontext32(®s, &frame->sf_sc); - if (sig < 0) - goto badframe; - else if (sig) - force_sig(sig, current); - - /* - * Don't let your children do this ... - */ - __asm__ __volatile__( - "move\t$29, %0\n\t" - "j\tsyscall_exit" - :/* no outputs */ - :"r" (®s)); - /* Unreached */ - -badframe: - force_sig(SIGSEGV, current); -} - -asmlinkage void sys32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) -{ - struct rt_sigframe32 __user *frame; - sigset_t set; - int sig; - - frame = (struct rt_sigframe32 __user *) regs.regs[29]; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) - goto badframe; - if (__copy_conv_sigset_from_user(&set, &frame->rs_uc.uc_sigmask)) - goto badframe; - - set_current_blocked(&set); - - sig = restore_sigcontext32(®s, &frame->rs_uc.uc_mcontext); - if (sig < 0) - goto badframe; - else if (sig) - force_sig(sig, current); - - if (compat_restore_altstack(&frame->rs_uc.uc_stack)) - goto badframe; - - /* - * Don't let your children do this ... - */ - __asm__ __volatile__( - "move\t$29, %0\n\t" - "j\tsyscall_exit" - :/* no outputs */ - :"r" (®s)); - /* Unreached */ - -badframe: - force_sig(SIGSEGV, current); -} - -static int setup_frame_32(void *sig_return, struct k_sigaction *ka, - struct pt_regs *regs, int signr, sigset_t *set) -{ - struct sigframe32 __user *frame; - int err = 0; - - frame = get_sigframe(ka, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame))) - goto give_sigsegv; - - err |= setup_sigcontext32(regs, &frame->sf_sc); - err |= __copy_conv_sigset_to_user(&frame->sf_mask, set); - - if (err) - goto give_sigsegv; - - /* - * Arguments to signal handler: - * - * a0 = signal number - * a1 = 0 (should be cause) - * a2 = pointer to struct sigcontext - * - * $25 and c0_epc point to the signal handler, $29 points to the - * struct sigframe. - */ - regs->regs[ 4] = signr; - regs->regs[ 5] = 0; - regs->regs[ 6] = (unsigned long) &frame->sf_sc; - regs->regs[29] = (unsigned long) frame; - regs->regs[31] = (unsigned long) sig_return; - regs->cp0_epc = regs->regs[25] = (unsigned long) ka->sa.sa_handler; - - DEBUGP("SIG deliver (%s:%d): sp=0x%p pc=0x%lx ra=0x%lx\n", - current->comm, current->pid, - frame, regs->cp0_epc, regs->regs[31]); - - return 0; - -give_sigsegv: - force_sigsegv(signr, current); - return -EFAULT; -} - -static int setup_rt_frame_32(void *sig_return, struct k_sigaction *ka, - struct pt_regs *regs, int signr, sigset_t *set, - siginfo_t *info) -{ - struct rt_sigframe32 __user *frame; - int err = 0; - - frame = get_sigframe(ka, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame))) - goto give_sigsegv; - - /* Convert (siginfo_t -> compat_siginfo_t) and copy to user. */ - err |= copy_siginfo_to_user32(&frame->rs_info, info); - - /* Create the ucontext. */ - err |= __put_user(0, &frame->rs_uc.uc_flags); - err |= __put_user(0, &frame->rs_uc.uc_link); - err |= __compat_save_altstack(&frame->rs_uc.uc_stack, regs->regs[29]); - err |= setup_sigcontext32(regs, &frame->rs_uc.uc_mcontext); - err |= __copy_conv_sigset_to_user(&frame->rs_uc.uc_sigmask, set); - - if (err) - goto give_sigsegv; - - /* - * Arguments to signal handler: - * - * a0 = signal number - * a1 = 0 (should be cause) - * a2 = pointer to ucontext - * - * $25 and c0_epc point to the signal handler, $29 points to - * the struct rt_sigframe32. - */ - regs->regs[ 4] = signr; - regs->regs[ 5] = (unsigned long) &frame->rs_info; - regs->regs[ 6] = (unsigned long) &frame->rs_uc; - regs->regs[29] = (unsigned long) frame; - regs->regs[31] = (unsigned long) sig_return; - regs->cp0_epc = regs->regs[25] = (unsigned long) ka->sa.sa_handler; - - DEBUGP("SIG deliver (%s:%d): sp=0x%p pc=0x%lx ra=0x%lx\n", - current->comm, current->pid, - frame, regs->cp0_epc, regs->regs[31]); - - return 0; - -give_sigsegv: - force_sigsegv(signr, current); - return -EFAULT; -} - -/* - * o32 compatibility on 64-bit kernels, without DSP ASE - */ -struct mips_abi mips_abi_32 = { - .setup_frame = setup_frame_32, - .signal_return_offset = - offsetof(struct mips_vdso, o32_signal_trampoline), - .setup_rt_frame = setup_rt_frame_32, - .rt_signal_return_offset = - offsetof(struct mips_vdso, o32_rt_signal_trampoline), - .restart = __NR_O32_restart_syscall -}; - -static int signal32_init(void) -{ - if (cpu_has_fpu) { - save_fp_context32 = _save_fp_context32; - restore_fp_context32 = _restore_fp_context32; - } else { - save_fp_context32 = fpu_emulator_save_context32; - restore_fp_context32 = fpu_emulator_restore_context32; - } - - return 0; -} - -arch_initcall(signal32_init); diff --git a/arch/mips/kernel/signal_n32.c b/arch/mips/kernel/signal_n32.c index b2241bb9cac1..139d2596b0d4 100644 --- a/arch/mips/kernel/signal_n32.c +++ b/arch/mips/kernel/signal_n32.c @@ -1,19 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2003 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ #include <linux/cache.h> #include <linux/sched.h> @@ -33,12 +20,11 @@ #include <asm/cacheflush.h> #include <asm/compat-signal.h> #include <asm/sim.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/ucontext.h> #include <asm/fpu.h> #include <asm/cpu-features.h> -#include <asm/war.h> -#include <asm/vdso.h> +#include <asm/syscalls.h> #include "signal-common.h" @@ -47,9 +33,6 @@ */ #define __NR_N32_restart_syscall 6214 -extern int setup_sigcontext(struct pt_regs *, struct sigcontext __user *); -extern int restore_sigcontext(struct pt_regs *, struct sigcontext __user *); - struct ucontextn32 { u32 uc_flags; s32 uc_link; @@ -65,25 +48,27 @@ struct rt_sigframe_n32 { struct ucontextn32 rs_uc; }; -asmlinkage void sysn32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) +asmlinkage void sysn32_rt_sigreturn(void) { struct rt_sigframe_n32 __user *frame; + struct pt_regs *regs; sigset_t set; int sig; - frame = (struct rt_sigframe_n32 __user *) regs.regs[29]; - if (!access_ok(VERIFY_READ, frame, sizeof(*frame))) + regs = current_pt_regs(); + frame = (struct rt_sigframe_n32 __user *)regs->regs[29]; + if (!access_ok(frame, sizeof(*frame))) goto badframe; if (__copy_conv_sigset_from_user(&set, &frame->rs_uc.uc_sigmask)) goto badframe; set_current_blocked(&set); - sig = restore_sigcontext(®s, &frame->rs_uc.uc_mcontext); + sig = restore_sigcontext(regs, &frame->rs_uc.uc_mcontext); if (sig < 0) goto badframe; else if (sig) - force_sig(sig, current); + force_sig(sig); if (compat_restore_altstack(&frame->rs_uc.uc_stack)) goto badframe; @@ -94,26 +79,26 @@ asmlinkage void sysn32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) __asm__ __volatile__( "move\t$29, %0\n\t" "j\tsyscall_exit" - :/* no outputs */ - :"r" (®s)); + : /* no outputs */ + : "r" (regs)); /* Unreached */ badframe: - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); } -static int setup_rt_frame_n32(void *sig_return, struct k_sigaction *ka, - struct pt_regs *regs, int signr, sigset_t *set, siginfo_t *info) +static int setup_rt_frame_n32(void *sig_return, struct ksignal *ksig, + struct pt_regs *regs, sigset_t *set) { struct rt_sigframe_n32 __user *frame; int err = 0; - frame = get_sigframe(ka, regs, sizeof(*frame)); - if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame))) - goto give_sigsegv; + frame = get_sigframe(ksig, regs, sizeof(*frame)); + if (!access_ok(frame, sizeof (*frame))) + return -EFAULT; /* Create siginfo. */ - err |= copy_siginfo_to_user32(&frame->rs_info, info); + err |= copy_siginfo_to_user32(&frame->rs_info, &ksig->info); /* Create the ucontext. */ err |= __put_user(0, &frame->rs_uc.uc_flags); @@ -123,7 +108,7 @@ static int setup_rt_frame_n32(void *sig_return, struct k_sigaction *ka, err |= __copy_conv_sigset_to_user(&frame->rs_uc.uc_sigmask, set); if (err) - goto give_sigsegv; + return -EFAULT; /* * Arguments to signal handler: @@ -135,27 +120,27 @@ static int setup_rt_frame_n32(void *sig_return, struct k_sigaction *ka, * $25 and c0_epc point to the signal handler, $29 points to * the struct rt_sigframe. */ - regs->regs[ 4] = signr; + regs->regs[ 4] = ksig->sig; regs->regs[ 5] = (unsigned long) &frame->rs_info; regs->regs[ 6] = (unsigned long) &frame->rs_uc; regs->regs[29] = (unsigned long) frame; regs->regs[31] = (unsigned long) sig_return; - regs->cp0_epc = regs->regs[25] = (unsigned long) ka->sa.sa_handler; + regs->cp0_epc = regs->regs[25] = (unsigned long) ksig->ka.sa.sa_handler; DEBUGP("SIG deliver (%s:%d): sp=0x%p pc=0x%lx ra=0x%lx\n", current->comm, current->pid, frame, regs->cp0_epc, regs->regs[31]); return 0; - -give_sigsegv: - force_sigsegv(signr, current); - return -EFAULT; } struct mips_abi mips_abi_n32 = { .setup_rt_frame = setup_rt_frame_n32, - .rt_signal_return_offset = - offsetof(struct mips_vdso, n32_rt_signal_trampoline), - .restart = __NR_N32_restart_syscall + .restart = __NR_N32_restart_syscall, + + .off_sc_fpregs = offsetof(struct sigcontext, sc_fpregs), + .off_sc_fpc_csr = offsetof(struct sigcontext, sc_fpc_csr), + .off_sc_used_math = offsetof(struct sigcontext, sc_used_math), + + .vdso = &vdso_image_n32, }; diff --git a/arch/mips/kernel/signal_o32.c b/arch/mips/kernel/signal_o32.c new file mode 100644 index 000000000000..4f0458459650 --- /dev/null +++ b/arch/mips/kernel/signal_o32.c @@ -0,0 +1,291 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 1991, 1992 Linus Torvalds + * Copyright (C) 1994 - 2000, 2006 Ralf Baechle + * Copyright (C) 1999, 2000 Silicon Graphics, Inc. + * Copyright (C) 2016, Imagination Technologies Ltd. + */ +#include <linux/compiler.h> +#include <linux/errno.h> +#include <linux/signal.h> +#include <linux/sched/signal.h> +#include <linux/uaccess.h> + +#include <asm/abi.h> +#include <asm/compat-signal.h> +#include <asm/dsp.h> +#include <asm/sim.h> +#include <asm/unistd.h> +#include <asm/syscalls.h> + +#include "signal-common.h" + +/* + * Including <asm/unistd.h> would give use the 64-bit syscall numbers ... + */ +#define __NR_O32_restart_syscall 4253 + +struct sigframe32 { + u32 sf_ass[4]; /* argument save space for o32 */ + u32 sf_pad[2]; /* Was: signal trampoline */ + struct sigcontext32 sf_sc; + compat_sigset_t sf_mask; +}; + +struct ucontext32 { + u32 uc_flags; + s32 uc_link; + compat_stack_t uc_stack; + struct sigcontext32 uc_mcontext; + compat_sigset_t uc_sigmask; /* mask last for extensibility */ +}; + +struct rt_sigframe32 { + u32 rs_ass[4]; /* argument save space for o32 */ + u32 rs_pad[2]; /* Was: signal trampoline */ + compat_siginfo_t rs_info; + struct ucontext32 rs_uc; +}; + +static int setup_sigcontext32(struct pt_regs *regs, + struct sigcontext32 __user *sc) +{ + int err = 0; + int i; + + err |= __put_user(regs->cp0_epc, &sc->sc_pc); + + err |= __put_user(0, &sc->sc_regs[0]); + for (i = 1; i < 32; i++) + err |= __put_user(regs->regs[i], &sc->sc_regs[i]); + + err |= __put_user(regs->hi, &sc->sc_mdhi); + err |= __put_user(regs->lo, &sc->sc_mdlo); + if (cpu_has_dsp) { + err |= __put_user(rddsp(DSP_MASK), &sc->sc_dsp); + err |= __put_user(mfhi1(), &sc->sc_hi1); + err |= __put_user(mflo1(), &sc->sc_lo1); + err |= __put_user(mfhi2(), &sc->sc_hi2); + err |= __put_user(mflo2(), &sc->sc_lo2); + err |= __put_user(mfhi3(), &sc->sc_hi3); + err |= __put_user(mflo3(), &sc->sc_lo3); + } + + /* + * Save FPU state to signal context. Signal handler + * will "inherit" current FPU state. + */ + err |= protected_save_fp_context(sc); + + return err; +} + +static int restore_sigcontext32(struct pt_regs *regs, + struct sigcontext32 __user *sc) +{ + int err = 0; + s32 treg; + int i; + + /* Always make any pending restarted system calls return -EINTR */ + current->restart_block.fn = do_no_restart_syscall; + + err |= __get_user(regs->cp0_epc, &sc->sc_pc); + err |= __get_user(regs->hi, &sc->sc_mdhi); + err |= __get_user(regs->lo, &sc->sc_mdlo); + if (cpu_has_dsp) { + err |= __get_user(treg, &sc->sc_hi1); mthi1(treg); + err |= __get_user(treg, &sc->sc_lo1); mtlo1(treg); + err |= __get_user(treg, &sc->sc_hi2); mthi2(treg); + err |= __get_user(treg, &sc->sc_lo2); mtlo2(treg); + err |= __get_user(treg, &sc->sc_hi3); mthi3(treg); + err |= __get_user(treg, &sc->sc_lo3); mtlo3(treg); + err |= __get_user(treg, &sc->sc_dsp); wrdsp(treg, DSP_MASK); + } + + for (i = 1; i < 32; i++) + err |= __get_user(regs->regs[i], &sc->sc_regs[i]); + + return err ?: protected_restore_fp_context(sc); +} + +static int setup_frame_32(void *sig_return, struct ksignal *ksig, + struct pt_regs *regs, sigset_t *set) +{ + struct sigframe32 __user *frame; + int err = 0; + + frame = get_sigframe(ksig, regs, sizeof(*frame)); + if (!access_ok(frame, sizeof (*frame))) + return -EFAULT; + + err |= setup_sigcontext32(regs, &frame->sf_sc); + err |= __copy_conv_sigset_to_user(&frame->sf_mask, set); + + if (err) + return -EFAULT; + + /* + * Arguments to signal handler: + * + * a0 = signal number + * a1 = 0 (should be cause) + * a2 = pointer to struct sigcontext + * + * $25 and c0_epc point to the signal handler, $29 points to the + * struct sigframe. + */ + regs->regs[ 4] = ksig->sig; + regs->regs[ 5] = 0; + regs->regs[ 6] = (unsigned long) &frame->sf_sc; + regs->regs[29] = (unsigned long) frame; + regs->regs[31] = (unsigned long) sig_return; + regs->cp0_epc = regs->regs[25] = (unsigned long) ksig->ka.sa.sa_handler; + + DEBUGP("SIG deliver (%s:%d): sp=0x%p pc=0x%lx ra=0x%lx\n", + current->comm, current->pid, + frame, regs->cp0_epc, regs->regs[31]); + + return 0; +} + +asmlinkage void sys32_rt_sigreturn(void) +{ + struct rt_sigframe32 __user *frame; + struct pt_regs *regs; + sigset_t set; + int sig; + + regs = current_pt_regs(); + frame = (struct rt_sigframe32 __user *)regs->regs[29]; + if (!access_ok(frame, sizeof(*frame))) + goto badframe; + if (__copy_conv_sigset_from_user(&set, &frame->rs_uc.uc_sigmask)) + goto badframe; + + set_current_blocked(&set); + + sig = restore_sigcontext32(regs, &frame->rs_uc.uc_mcontext); + if (sig < 0) + goto badframe; + else if (sig) + force_sig(sig); + + if (compat_restore_altstack(&frame->rs_uc.uc_stack)) + goto badframe; + + /* + * Don't let your children do this ... + */ + __asm__ __volatile__( + "move\t$29, %0\n\t" + "j\tsyscall_exit" + : /* no outputs */ + : "r" (regs)); + /* Unreached */ + +badframe: + force_sig(SIGSEGV); +} + +static int setup_rt_frame_32(void *sig_return, struct ksignal *ksig, + struct pt_regs *regs, sigset_t *set) +{ + struct rt_sigframe32 __user *frame; + int err = 0; + + frame = get_sigframe(ksig, regs, sizeof(*frame)); + if (!access_ok(frame, sizeof (*frame))) + return -EFAULT; + + /* Convert (siginfo_t -> compat_siginfo_t) and copy to user. */ + err |= copy_siginfo_to_user32(&frame->rs_info, &ksig->info); + + /* Create the ucontext. */ + err |= __put_user(0, &frame->rs_uc.uc_flags); + err |= __put_user(0, &frame->rs_uc.uc_link); + err |= __compat_save_altstack(&frame->rs_uc.uc_stack, regs->regs[29]); + err |= setup_sigcontext32(regs, &frame->rs_uc.uc_mcontext); + err |= __copy_conv_sigset_to_user(&frame->rs_uc.uc_sigmask, set); + + if (err) + return -EFAULT; + + /* + * Arguments to signal handler: + * + * a0 = signal number + * a1 = 0 (should be cause) + * a2 = pointer to ucontext + * + * $25 and c0_epc point to the signal handler, $29 points to + * the struct rt_sigframe32. + */ + regs->regs[ 4] = ksig->sig; + regs->regs[ 5] = (unsigned long) &frame->rs_info; + regs->regs[ 6] = (unsigned long) &frame->rs_uc; + regs->regs[29] = (unsigned long) frame; + regs->regs[31] = (unsigned long) sig_return; + regs->cp0_epc = regs->regs[25] = (unsigned long) ksig->ka.sa.sa_handler; + + DEBUGP("SIG deliver (%s:%d): sp=0x%p pc=0x%lx ra=0x%lx\n", + current->comm, current->pid, + frame, regs->cp0_epc, regs->regs[31]); + + return 0; +} + +/* + * o32 compatibility on 64-bit kernels, without DSP ASE + */ +struct mips_abi mips_abi_32 = { + .setup_frame = setup_frame_32, + .setup_rt_frame = setup_rt_frame_32, + .restart = __NR_O32_restart_syscall, + + .off_sc_fpregs = offsetof(struct sigcontext32, sc_fpregs), + .off_sc_fpc_csr = offsetof(struct sigcontext32, sc_fpc_csr), + .off_sc_used_math = offsetof(struct sigcontext32, sc_used_math), + + .vdso = &vdso_image_o32, +}; + + +asmlinkage void sys32_sigreturn(void) +{ + struct sigframe32 __user *frame; + struct pt_regs *regs; + sigset_t blocked; + int sig; + + regs = current_pt_regs(); + frame = (struct sigframe32 __user *)regs->regs[29]; + if (!access_ok(frame, sizeof(*frame))) + goto badframe; + if (__copy_conv_sigset_from_user(&blocked, &frame->sf_mask)) + goto badframe; + + set_current_blocked(&blocked); + + sig = restore_sigcontext32(regs, &frame->sf_sc); + if (sig < 0) + goto badframe; + else if (sig) + force_sig(sig); + + /* + * Don't let your children do this ... + */ + __asm__ __volatile__( + "move\t$29, %0\n\t" + "j\tsyscall_exit" + : /* no outputs */ + : "r" (regs)); + /* Unreached */ + +badframe: + force_sig(SIGSEGV); +} diff --git a/arch/mips/kernel/smp-bmips.c b/arch/mips/kernel/smp-bmips.c index aea6c0885838..35b8d810833c 100644 --- a/arch/mips/kernel/smp-bmips.c +++ b/arch/mips/kernel/smp-bmips.c @@ -10,6 +10,8 @@ #include <linux/init.h> #include <linux/sched.h> +#include <linux/sched/hotplug.h> +#include <linux/sched/task_stack.h> #include <linux/mm.h> #include <linux/delay.h> #include <linux/smp.h> @@ -23,18 +25,19 @@ #include <linux/linkage.h> #include <linux/bug.h> #include <linux/kernel.h> +#include <linux/kexec.h> +#include <linux/irq.h> #include <asm/time.h> -#include <asm/pgtable.h> #include <asm/processor.h> #include <asm/bootinfo.h> -#include <asm/pmon.h> #include <asm/cacheflush.h> #include <asm/tlbflush.h> #include <asm/mipsregs.h> #include <asm/bmips.h> #include <asm/traps.h> #include <asm/barrier.h> +#include <asm/cpu-features.h> static int __maybe_unused max_cpus = 1; @@ -42,15 +45,25 @@ static int __maybe_unused max_cpus = 1; int bmips_smp_enabled = 1; int bmips_cpu_offset; cpumask_t bmips_booted_mask; +unsigned long bmips_tp1_irqs = IE_IRQ1; + +#define RESET_FROM_KSEG0 0x80080800 +#define RESET_FROM_KSEG1 0xa0080800 + +static void bmips_set_reset_vec(int cpu, u32 val); #ifdef CONFIG_SMP +#include <asm/smp.h> + /* initial $sp, $gp - used by arch/mips/kernel/bmips_vec.S */ unsigned long bmips_smp_boot_sp; unsigned long bmips_smp_boot_gp; -static void bmips_send_ipi_single(int cpu, unsigned int action); -static irqreturn_t bmips_ipi_interrupt(int irq, void *dev_id); +static void bmips43xx_send_ipi_single(int cpu, unsigned int action); +static void bmips5000_send_ipi_single(int cpu, unsigned int action); +static irqreturn_t bmips43xx_ipi_interrupt(int irq, void *dev_id); +static irqreturn_t bmips5000_ipi_interrupt(int irq, void *dev_id); /* SW interrupts 0,1 are used for interprocessor signaling */ #define IPI0_IRQ (MIPS_CPU_IRQ_BASE + 0) @@ -64,49 +77,58 @@ static irqreturn_t bmips_ipi_interrupt(int irq, void *dev_id); static void __init bmips_smp_setup(void) { int i, cpu = 1, boot_cpu = 0; + int cpu_hw_intr; + + switch (current_cpu_type()) { + case CPU_BMIPS4350: + case CPU_BMIPS4380: + /* arbitration priority */ + clear_c0_brcm_cmt_ctrl(0x30); + + /* NBK and weak order flags */ + set_c0_brcm_config_0(0x30000); + + /* Find out if we are running on TP0 or TP1 */ + boot_cpu = !!(read_c0_brcm_cmt_local() & (1 << 31)); + + /* + * MIPS interrupts 0,1 (SW INT 0,1) cross over to the other + * thread + * MIPS interrupt 2 (HW INT 0) is the CPU0 L1 controller output + * MIPS interrupt 3 (HW INT 1) is the CPU1 L1 controller output + */ + if (boot_cpu == 0) + cpu_hw_intr = 0x02; + else + cpu_hw_intr = 0x1d; -#if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380) - /* arbitration priority */ - clear_c0_brcm_cmt_ctrl(0x30); + change_c0_brcm_cmt_intr(0xf8018000, + (cpu_hw_intr << 27) | (0x03 << 15)); - /* NBK and weak order flags */ - set_c0_brcm_config_0(0x30000); + /* single core, 2 threads (2 pipelines) */ + max_cpus = 2; - /* Find out if we are running on TP0 or TP1 */ - boot_cpu = !!(read_c0_brcm_cmt_local() & (1 << 31)); + break; + case CPU_BMIPS5000: + /* enable raceless SW interrupts */ + set_c0_brcm_config(0x03 << 22); - /* - * MIPS interrupts 0,1 (SW INT 0,1) cross over to the other thread - * MIPS interrupt 2 (HW INT 0) is the CPU0 L1 controller output - * MIPS interrupt 3 (HW INT 1) is the CPU1 L1 controller output - * - * If booting from TP1, leave the existing CMT interrupt routing - * such that TP0 responds to SW1 and TP1 responds to SW0. - */ - if (boot_cpu == 0) - change_c0_brcm_cmt_intr(0xf8018000, - (0x02 << 27) | (0x03 << 15)); - else - change_c0_brcm_cmt_intr(0xf8018000, (0x1d << 27)); + /* route HW interrupt 0 to CPU0, HW interrupt 1 to CPU1 */ + change_c0_brcm_mode(0x1f << 27, 0x02 << 27); - /* single core, 2 threads (2 pipelines) */ - max_cpus = 2; -#elif defined(CONFIG_CPU_BMIPS5000) - /* enable raceless SW interrupts */ - set_c0_brcm_config(0x03 << 22); + /* N cores, 2 threads per core */ + max_cpus = (((read_c0_brcm_config() >> 6) & 0x03) + 1) << 1; - /* route HW interrupt 0 to CPU0, HW interrupt 1 to CPU1 */ - change_c0_brcm_mode(0x1f << 27, 0x02 << 27); - - /* N cores, 2 threads per core */ - max_cpus = (((read_c0_brcm_config() >> 6) & 0x03) + 1) << 1; + /* clear any pending SW interrupts */ + for (i = 0; i < max_cpus; i++) { + write_c0_brcm_action(ACTION_CLR_IPI(i, 0)); + write_c0_brcm_action(ACTION_CLR_IPI(i, 1)); + } - /* clear any pending SW interrupts */ - for (i = 0; i < max_cpus; i++) { - write_c0_brcm_action(ACTION_CLR_IPI(i, 0)); - write_c0_brcm_action(ACTION_CLR_IPI(i, 1)); + break; + default: + max_cpus = 1; } -#endif if (!bmips_smp_enabled) max_cpus = 1; @@ -115,17 +137,24 @@ static void __init bmips_smp_setup(void) if (!board_ebase_setup) board_ebase_setup = &bmips_ebase_setup; - __cpu_number_map[boot_cpu] = 0; - __cpu_logical_map[0] = boot_cpu; - - for (i = 0; i < max_cpus; i++) { - if (i != boot_cpu) { - __cpu_number_map[i] = cpu; - __cpu_logical_map[cpu] = i; - cpu++; + if (max_cpus > 1) { + __cpu_number_map[boot_cpu] = 0; + __cpu_logical_map[0] = boot_cpu; + + for (i = 0; i < max_cpus; i++) { + if (i != boot_cpu) { + __cpu_number_map[i] = cpu; + __cpu_logical_map[cpu] = i; + cpu++; + } + set_cpu_possible(i, 1); + set_cpu_present(i, 1); } - set_cpu_possible(i, 1); - set_cpu_present(i, 1); + } else { + __cpu_number_map[0] = boot_cpu; + __cpu_logical_map[0] = 0; + set_cpu_possible(0, 1); + set_cpu_present(0, 1); } } @@ -134,18 +163,32 @@ static void __init bmips_smp_setup(void) */ static void bmips_prepare_cpus(unsigned int max_cpus) { - if (request_irq(IPI0_IRQ, bmips_ipi_interrupt, IRQF_PERCPU, - "smp_ipi0", NULL)) - panic("Can't request IPI0 interrupt\n"); - if (request_irq(IPI1_IRQ, bmips_ipi_interrupt, IRQF_PERCPU, - "smp_ipi1", NULL)) - panic("Can't request IPI1 interrupt\n"); + irqreturn_t (*bmips_ipi_interrupt)(int irq, void *dev_id); + + switch (current_cpu_type()) { + case CPU_BMIPS4350: + case CPU_BMIPS4380: + bmips_ipi_interrupt = bmips43xx_ipi_interrupt; + break; + case CPU_BMIPS5000: + bmips_ipi_interrupt = bmips5000_ipi_interrupt; + break; + default: + return; + } + + if (request_irq(IPI0_IRQ, bmips_ipi_interrupt, + IRQF_PERCPU | IRQF_NO_SUSPEND, "smp_ipi0", NULL)) + panic("Can't request IPI0 interrupt"); + if (request_irq(IPI1_IRQ, bmips_ipi_interrupt, + IRQF_PERCPU | IRQF_NO_SUSPEND, "smp_ipi1", NULL)) + panic("Can't request IPI1 interrupt"); } /* * Tell the hardware to boot CPUx - runs on CPU0 */ -static void bmips_boot_secondary(int cpu, struct task_struct *idle) +static int bmips_boot_secondary(int cpu, struct task_struct *idle) { bmips_smp_boot_sp = __KSTK_TOS(idle); bmips_smp_boot_gp = (unsigned long)task_thread_info(idle); @@ -168,28 +211,37 @@ static void bmips_boot_secondary(int cpu, struct task_struct *idle) pr_info("SMP: Booting CPU%d...\n", cpu); - if (cpumask_test_cpu(cpu, &bmips_booted_mask)) - bmips_send_ipi_single(cpu, 0); - else { -#if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380) - /* Reset slave TP1 if booting from TP0 */ - if (cpu_logical_map(cpu) == 0) - set_c0_brcm_cmt_ctrl(0x01); -#elif defined(CONFIG_CPU_BMIPS5000) - if (cpu & 0x01) + if (cpumask_test_cpu(cpu, &bmips_booted_mask)) { + /* kseg1 might not exist if this CPU enabled XKS01 */ + bmips_set_reset_vec(cpu, RESET_FROM_KSEG0); + + switch (current_cpu_type()) { + case CPU_BMIPS4350: + case CPU_BMIPS4380: + bmips43xx_send_ipi_single(cpu, 0); + break; + case CPU_BMIPS5000: + bmips5000_send_ipi_single(cpu, 0); + break; + } + } else { + bmips_set_reset_vec(cpu, RESET_FROM_KSEG1); + + switch (current_cpu_type()) { + case CPU_BMIPS4350: + case CPU_BMIPS4380: + /* Reset slave TP1 if booting from TP0 */ + if (cpu_logical_map(cpu) == 1) + set_c0_brcm_cmt_ctrl(0x01); + break; + case CPU_BMIPS5000: write_c0_brcm_action(ACTION_BOOT_THREAD(cpu)); - else { - /* - * core N thread 0 was already booted; just - * pulse the NMI line - */ - bmips_write_zscm_reg(0x210, 0xc0000000); - udelay(10); - bmips_write_zscm_reg(0x210, 0x00); + break; } -#endif cpumask_set_cpu(cpu, &bmips_booted_mask); } + + return 0; } /* @@ -197,22 +249,18 @@ static void bmips_boot_secondary(int cpu, struct task_struct *idle) */ static void bmips_init_secondary(void) { - /* move NMI vector to kseg0, in case XKS01 is enabled */ - -#if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380) - void __iomem *cbr = BMIPS_GET_CBR(); - unsigned long old_vec; - - old_vec = __raw_readl(cbr + BMIPS_RELO_VECTOR_CONTROL_1); - __raw_writel(old_vec & ~0x20000000, cbr + BMIPS_RELO_VECTOR_CONTROL_1); - - clear_c0_cause(smp_processor_id() ? C_SW1 : C_SW0); -#elif defined(CONFIG_CPU_BMIPS5000) - write_c0_brcm_bootvec(read_c0_brcm_bootvec() & - (smp_processor_id() & 0x01 ? ~0x20000000 : ~0x2000)); - - write_c0_brcm_action(ACTION_CLR_IPI(smp_processor_id(), 0)); -#endif + bmips_cpu_setup(); + + switch (current_cpu_type()) { + case CPU_BMIPS4350: + case CPU_BMIPS4380: + clear_c0_cause(smp_processor_id() ? C_SW1 : C_SW0); + break; + case CPU_BMIPS5000: + write_c0_brcm_action(ACTION_CLR_IPI(smp_processor_id(), 0)); + cpu_set_core(¤t_cpu_data, (read_c0_brcm_config() >> 25) & 3); + break; + } } /* @@ -226,20 +274,11 @@ static void bmips_smp_finish(void) write_c0_compare(read_c0_count() + mips_hpt_frequency / HZ); irq_enable_hazard(); - set_c0_status(IE_SW0 | IE_SW1 | IE_IRQ1 | IE_IRQ5 | ST0_IE); + set_c0_status(IE_SW0 | IE_SW1 | bmips_tp1_irqs | IE_IRQ5 | ST0_IE); irq_enable_hazard(); } /* - * Runs on CPU0 after all CPUs have been booted - */ -static void bmips_cpus_done(void) -{ -} - -#if defined(CONFIG_CPU_BMIPS5000) - -/* * BMIPS5000 raceless IPIs * * Each CPU has two inbound SW IRQs which are independent of all other CPUs. @@ -247,12 +286,12 @@ static void bmips_cpus_done(void) * IPI1 is used for SMP_CALL_FUNCTION */ -static void bmips_send_ipi_single(int cpu, unsigned int action) +static void bmips5000_send_ipi_single(int cpu, unsigned int action) { write_c0_brcm_action(ACTION_SET_IPI(cpu, action == SMP_CALL_FUNCTION)); } -static irqreturn_t bmips_ipi_interrupt(int irq, void *dev_id) +static irqreturn_t bmips5000_ipi_interrupt(int irq, void *dev_id) { int action = irq - IPI0_IRQ; @@ -261,12 +300,19 @@ static irqreturn_t bmips_ipi_interrupt(int irq, void *dev_id) if (action == 0) scheduler_ipi(); else - smp_call_function_interrupt(); + generic_smp_call_function_interrupt(); return IRQ_HANDLED; } -#else +static void bmips5000_send_ipi_mask(const struct cpumask *mask, + unsigned int action) +{ + unsigned int i; + + for_each_cpu(i, mask) + bmips5000_send_ipi_single(i, action); +} /* * BMIPS43xx racey IPIs @@ -281,7 +327,7 @@ static irqreturn_t bmips_ipi_interrupt(int irq, void *dev_id) static DEFINE_SPINLOCK(ipi_lock); static DEFINE_PER_CPU(int, ipi_action_mask); -static void bmips_send_ipi_single(int cpu, unsigned int action) +static void bmips43xx_send_ipi_single(int cpu, unsigned int action) { unsigned long flags; @@ -292,13 +338,13 @@ static void bmips_send_ipi_single(int cpu, unsigned int action) spin_unlock_irqrestore(&ipi_lock, flags); } -static irqreturn_t bmips_ipi_interrupt(int irq, void *dev_id) +static irqreturn_t bmips43xx_ipi_interrupt(int irq, void *dev_id) { unsigned long flags; int action, cpu = irq - IPI0_IRQ; spin_lock_irqsave(&ipi_lock, flags); - action = __get_cpu_var(ipi_action_mask); + action = __this_cpu_read(ipi_action_mask); per_cpu(ipi_action_mask, cpu) = 0; clear_c0_cause(cpu ? C_SW1 : C_SW0); spin_unlock_irqrestore(&ipi_lock, flags); @@ -306,20 +352,18 @@ static irqreturn_t bmips_ipi_interrupt(int irq, void *dev_id) if (action & SMP_RESCHEDULE_YOURSELF) scheduler_ipi(); if (action & SMP_CALL_FUNCTION) - smp_call_function_interrupt(); + generic_smp_call_function_interrupt(); return IRQ_HANDLED; } -#endif /* BMIPS type */ - -static void bmips_send_ipi_mask(const struct cpumask *mask, +static void bmips43xx_send_ipi_mask(const struct cpumask *mask, unsigned int action) { unsigned int i; for_each_cpu(i, mask) - bmips_send_ipi_single(i, action); + bmips43xx_send_ipi_single(i, action); } #ifdef CONFIG_HOTPLUG_CPU @@ -328,13 +372,12 @@ static int bmips_cpu_disable(void) { unsigned int cpu = smp_processor_id(); - if (cpu == 0) - return -EBUSY; - pr_info("SMP: CPU%d is offline\n", cpu); set_cpu_online(cpu, false); - cpu_clear(cpu, cpu_callin_map); + calculate_cpu_foreign_map(); + irq_migrate_all_off_this_cpu(); + clear_c0_status(IE_IRQ5); local_flush_tlb_all(); local_flush_icache_range(0, ~0); @@ -349,6 +392,7 @@ static void bmips_cpu_die(unsigned int cpu) void __ref play_dead(void) { idle_task_exit(); + cpuhp_ap_report_dead(); /* flush data cache */ _dma_cache_wback_inv(0, ~0); @@ -359,7 +403,8 @@ void __ref play_dead(void) * IRQ handlers; this clears ST0_IE and returns immediately. */ clear_c0_cause(CAUSEF_IV | C_SW0 | C_SW1); - change_c0_status(IE_IRQ5 | IE_IRQ1 | IE_SW0 | IE_SW1 | ST0_IE | ST0_BEV, + change_c0_status( + IE_IRQ5 | bmips_tp1_irqs | IE_SW0 | IE_SW1 | ST0_IE | ST0_BEV, IE_SW0 | IE_SW1 | ST0_IE | ST0_BEV); irq_disable_hazard(); @@ -371,23 +416,44 @@ void __ref play_dead(void) " wait\n" " j bmips_secondary_reentry\n" : : : "memory"); + + BUG(); } #endif /* CONFIG_HOTPLUG_CPU */ -struct plat_smp_ops bmips_smp_ops = { +const struct plat_smp_ops bmips43xx_smp_ops = { + .smp_setup = bmips_smp_setup, + .prepare_cpus = bmips_prepare_cpus, + .boot_secondary = bmips_boot_secondary, + .smp_finish = bmips_smp_finish, + .init_secondary = bmips_init_secondary, + .send_ipi_single = bmips43xx_send_ipi_single, + .send_ipi_mask = bmips43xx_send_ipi_mask, +#ifdef CONFIG_HOTPLUG_CPU + .cpu_disable = bmips_cpu_disable, + .cpu_die = bmips_cpu_die, +#endif +#ifdef CONFIG_KEXEC_CORE + .kexec_nonboot_cpu = kexec_nonboot_cpu_jump, +#endif +}; + +const struct plat_smp_ops bmips5000_smp_ops = { .smp_setup = bmips_smp_setup, .prepare_cpus = bmips_prepare_cpus, .boot_secondary = bmips_boot_secondary, .smp_finish = bmips_smp_finish, .init_secondary = bmips_init_secondary, - .cpus_done = bmips_cpus_done, - .send_ipi_single = bmips_send_ipi_single, - .send_ipi_mask = bmips_send_ipi_mask, + .send_ipi_single = bmips5000_send_ipi_single, + .send_ipi_mask = bmips5000_send_ipi_mask, #ifdef CONFIG_HOTPLUG_CPU .cpu_disable = bmips_cpu_disable, .cpu_die = bmips_cpu_die, #endif +#ifdef CONFIG_KEXEC_CORE + .kexec_nonboot_cpu = kexec_nonboot_cpu_jump, +#endif }; #endif /* CONFIG_SMP */ @@ -398,66 +464,118 @@ struct plat_smp_ops bmips_smp_ops = { * UP BMIPS systems as well. ***********************************************************************/ -static void __cpuinit bmips_wr_vec(unsigned long dst, char *start, char *end) +static void bmips_wr_vec(unsigned long dst, char *start, char *end) { memcpy((void *)dst, start, end - start); - dma_cache_wback((unsigned long)start, end - start); + dma_cache_wback(dst, end - start); local_flush_icache_range(dst, dst + (end - start)); instruction_hazard(); } -static inline void __cpuinit bmips_nmi_handler_setup(void) +static inline void bmips_nmi_handler_setup(void) { - bmips_wr_vec(BMIPS_NMI_RESET_VEC, &bmips_reset_nmi_vec, - &bmips_reset_nmi_vec_end); - bmips_wr_vec(BMIPS_WARM_RESTART_VEC, &bmips_smp_int_vec, - &bmips_smp_int_vec_end); + bmips_wr_vec(BMIPS_NMI_RESET_VEC, bmips_reset_nmi_vec, + bmips_reset_nmi_vec_end); + bmips_wr_vec(BMIPS_WARM_RESTART_VEC, bmips_smp_int_vec, + bmips_smp_int_vec_end); } -void __cpuinit bmips_ebase_setup(void) +struct reset_vec_info { + int cpu; + u32 val; +}; + +static void bmips_set_reset_vec_remote(void *vinfo) +{ + struct reset_vec_info *info = vinfo; + int shift = info->cpu & 0x01 ? 16 : 0; + u32 mask = ~(0xffff << shift), val = info->val >> 16; + + preempt_disable(); + if (smp_processor_id() > 0) { + smp_call_function_single(0, &bmips_set_reset_vec_remote, + info, 1); + } else { + if (info->cpu & 0x02) { + /* BMIPS5200 "should" use mask/shift, but it's buggy */ + bmips_write_zscm_reg(0xa0, (val << 16) | val); + bmips_read_zscm_reg(0xa0); + } else { + write_c0_brcm_bootvec((read_c0_brcm_bootvec() & mask) | + (val << shift)); + } + } + preempt_enable(); +} + +static void bmips_set_reset_vec(int cpu, u32 val) +{ + struct reset_vec_info info; + + if (current_cpu_type() == CPU_BMIPS5000) { + /* this needs to run from CPU0 (which is always online) */ + info.cpu = cpu; + info.val = val; + bmips_set_reset_vec_remote(&info); + } else { + void __iomem *cbr = bmips_cbr_addr; + + if (cpu == 0) + __raw_writel(val, cbr + BMIPS_RELO_VECTOR_CONTROL_0); + else { + if (current_cpu_type() != CPU_BMIPS4380) + return; + __raw_writel(val, cbr + BMIPS_RELO_VECTOR_CONTROL_1); + } + } + __sync(); + back_to_back_c0_hazard(); +} + +void bmips_ebase_setup(void) { unsigned long new_ebase = ebase; - void __iomem __maybe_unused *cbr; BUG_ON(ebase != CKSEG0); -#if defined(CONFIG_CPU_BMIPS4350) - /* - * BMIPS4350 cannot relocate the normal vectors, but it - * can relocate the BEV=1 vectors. So CPU1 starts up at - * the relocated BEV=1, IV=0 general exception vector @ - * 0xa000_0380. - * - * set_uncached_handler() is used here because: - * - CPU1 will run this from uncached space - * - None of the cacheflush functions are set up yet - */ - set_uncached_handler(BMIPS_WARM_RESTART_VEC - CKSEG0, - &bmips_smp_int_vec, 0x80); - __sync(); - return; -#elif defined(CONFIG_CPU_BMIPS4380) - /* - * 0x8000_0000: reset/NMI (initially in kseg1) - * 0x8000_0400: normal vectors - */ - new_ebase = 0x80000400; - cbr = BMIPS_GET_CBR(); - __raw_writel(0x80080800, cbr + BMIPS_RELO_VECTOR_CONTROL_0); - __raw_writel(0xa0080800, cbr + BMIPS_RELO_VECTOR_CONTROL_1); -#elif defined(CONFIG_CPU_BMIPS5000) - /* - * 0x8000_0000: reset/NMI (initially in kseg1) - * 0x8000_1000: normal vectors - */ - new_ebase = 0x80001000; - write_c0_brcm_bootvec(0xa0088008); - write_c0_ebase(new_ebase); - if (max_cpus > 2) - bmips_write_zscm_reg(0xa0, 0xa008a008); -#else - return; -#endif + switch (current_cpu_type()) { + case CPU_BMIPS4350: + /* + * BMIPS4350 cannot relocate the normal vectors, but it + * can relocate the BEV=1 vectors. So CPU1 starts up at + * the relocated BEV=1, IV=0 general exception vector @ + * 0xa000_0380. + * + * set_uncached_handler() is used here because: + * - CPU1 will run this from uncached space + * - None of the cacheflush functions are set up yet + */ + set_uncached_handler(BMIPS_WARM_RESTART_VEC - CKSEG0, + &bmips_smp_int_vec, 0x80); + __sync(); + return; + case CPU_BMIPS3300: + case CPU_BMIPS4380: + /* + * 0x8000_0000: reset/NMI (initially in kseg1) + * 0x8000_0400: normal vectors + */ + new_ebase = 0x80000400; + bmips_set_reset_vec(0, RESET_FROM_KSEG0); + break; + case CPU_BMIPS5000: + /* + * 0x8000_0000: reset/NMI (initially in kseg1) + * 0x8000_1000: normal vectors + */ + new_ebase = 0x80001000; + bmips_set_reset_vec(0, RESET_FROM_KSEG0); + write_c0_ebase(new_ebase); + break; + default: + return; + } + board_nmi_handler_setup = &bmips_nmi_handler_setup; ebase = new_ebase; } @@ -470,3 +588,108 @@ asmlinkage void __weak plat_wired_tlb_setup(void) * once the wired entries are present. */ } + +void bmips_cpu_setup(void) +{ + void __iomem __maybe_unused *cbr = bmips_cbr_addr; + u32 __maybe_unused rac_addr; + u32 __maybe_unused cfg; + + switch (current_cpu_type()) { + case CPU_BMIPS3300: + /* Set BIU to async mode */ + set_c0_brcm_bus_pll(BIT(22)); + __sync(); + + /* put the BIU back in sync mode */ + clear_c0_brcm_bus_pll(BIT(22)); + + /* clear BHTD to enable branch history table */ + clear_c0_brcm_reset(BIT(16)); + + /* Flush and enable RAC */ + cfg = __raw_readl(cbr + BMIPS_RAC_CONFIG); + __raw_writel(cfg | 0x100, cbr + BMIPS_RAC_CONFIG); + __raw_readl(cbr + BMIPS_RAC_CONFIG); + + cfg = __raw_readl(cbr + BMIPS_RAC_CONFIG); + __raw_writel(cfg | 0xf, cbr + BMIPS_RAC_CONFIG); + __raw_readl(cbr + BMIPS_RAC_CONFIG); + + cfg = __raw_readl(cbr + BMIPS_RAC_ADDRESS_RANGE); + __raw_writel(cfg | 0x0fff0000, cbr + BMIPS_RAC_ADDRESS_RANGE); + __raw_readl(cbr + BMIPS_RAC_ADDRESS_RANGE); + break; + + case CPU_BMIPS4350: + rac_addr = BMIPS_RAC_CONFIG_1; + + if (!(read_c0_brcm_cmt_local() & (1 << 31))) + rac_addr = BMIPS_RAC_CONFIG; + + /* Enable data RAC */ + cfg = __raw_readl(cbr + rac_addr); + __raw_writel(cfg | 0xf, cbr + rac_addr); + __raw_readl(cbr + rac_addr); + + /* Flush stale data out of the readahead cache */ + cfg = __raw_readl(cbr + BMIPS_RAC_CONFIG); + __raw_writel(cfg | 0x100, cbr + BMIPS_RAC_CONFIG); + __raw_readl(cbr + BMIPS_RAC_CONFIG); + break; + + case CPU_BMIPS4380: + /* CBG workaround for early BMIPS4380 CPUs */ + switch (read_c0_prid()) { + case 0x2a040: + case 0x2a042: + case 0x2a044: + case 0x2a060: + cfg = __raw_readl(cbr + BMIPS_L2_CONFIG); + __raw_writel(cfg & ~0x07000000, cbr + BMIPS_L2_CONFIG); + __raw_readl(cbr + BMIPS_L2_CONFIG); + } + + /* clear BHTD to enable branch history table */ + clear_c0_brcm_config_0(BIT(21)); + + /* XI/ROTR enable */ + set_c0_brcm_config_0(BIT(23)); + set_c0_brcm_cmt_ctrl(BIT(15)); + break; + + case CPU_BMIPS5000: + /* enable RDHWR, BRDHWR */ + set_c0_brcm_config(BIT(17) | BIT(21)); + + /* Disable JTB */ + __asm__ __volatile__( + " .set noreorder\n" + " li $8, 0x5a455048\n" + " .word 0x4088b00f\n" /* mtc0 t0, $22, 15 */ + " .word 0x4008b008\n" /* mfc0 t0, $22, 8 */ + " li $9, 0x00008000\n" + " or $8, $8, $9\n" + " .word 0x4088b008\n" /* mtc0 t0, $22, 8 */ + " sync\n" + " li $8, 0x0\n" + " .word 0x4088b00f\n" /* mtc0 t0, $22, 15 */ + " .set reorder\n" + : : : "$8", "$9"); + + /* XI enable */ + set_c0_brcm_config(BIT(27)); + + /* enable MIPS32R2 ROR instruction for XI TLB handlers */ + __asm__ __volatile__( + " li $8, 0x5a455048\n" + " .word 0x4088b00f\n" /* mtc0 $8, $22, 15 */ + " nop; nop; nop\n" + " .word 0x4008b008\n" /* mfc0 $8, $22, 8 */ + " lui $9, 0x0100\n" + " or $8, $9\n" + " .word 0x4088b008\n" /* mtc0 $8, $22, 8 */ + : : : "$8", "$9"); + break; + } +} diff --git a/arch/mips/kernel/smp-cmp.c b/arch/mips/kernel/smp-cmp.c deleted file mode 100644 index c2e5d74739b4..000000000000 --- a/arch/mips/kernel/smp-cmp.c +++ /dev/null @@ -1,209 +0,0 @@ -/* - * This program is free software; you can distribute it and/or modify it - * under the terms of the GNU General Public License (Version 2) as - * published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. - * - * Copyright (C) 2007 MIPS Technologies, Inc. - * Chris Dearman (chris@mips.com) - */ - -#undef DEBUG - -#include <linux/kernel.h> -#include <linux/sched.h> -#include <linux/smp.h> -#include <linux/cpumask.h> -#include <linux/interrupt.h> -#include <linux/compiler.h> - -#include <linux/atomic.h> -#include <asm/cacheflush.h> -#include <asm/cpu.h> -#include <asm/processor.h> -#include <asm/hardirq.h> -#include <asm/mmu_context.h> -#include <asm/smp.h> -#include <asm/time.h> -#include <asm/mipsregs.h> -#include <asm/mipsmtregs.h> -#include <asm/mips_mt.h> -#include <asm/amon.h> -#include <asm/gic.h> - -static void ipi_call_function(unsigned int cpu) -{ - pr_debug("CPU%d: %s cpu %d status %08x\n", - smp_processor_id(), __func__, cpu, read_c0_status()); - - gic_send_ipi(plat_ipi_call_int_xlate(cpu)); -} - - -static void ipi_resched(unsigned int cpu) -{ - pr_debug("CPU%d: %s cpu %d status %08x\n", - smp_processor_id(), __func__, cpu, read_c0_status()); - - gic_send_ipi(plat_ipi_resched_int_xlate(cpu)); -} - -/* - * FIXME: This isn't restricted to CMP - * The SMVP kernel could use GIC interrupts if available - */ -void cmp_send_ipi_single(int cpu, unsigned int action) -{ - unsigned long flags; - - local_irq_save(flags); - - switch (action) { - case SMP_CALL_FUNCTION: - ipi_call_function(cpu); - break; - - case SMP_RESCHEDULE_YOURSELF: - ipi_resched(cpu); - break; - } - - local_irq_restore(flags); -} - -static void cmp_send_ipi_mask(const struct cpumask *mask, unsigned int action) -{ - unsigned int i; - - for_each_cpu(i, mask) - cmp_send_ipi_single(i, action); -} - -static void cmp_init_secondary(void) -{ - struct cpuinfo_mips *c = ¤t_cpu_data; - - /* Assume GIC is present */ - change_c0_status(ST0_IM, STATUSF_IP3 | STATUSF_IP4 | STATUSF_IP6 | - STATUSF_IP7); - - /* Enable per-cpu interrupts: platform specific */ - - c->core = (read_c0_ebase() >> 1) & 0x1ff; -#if defined(CONFIG_MIPS_MT_SMP) || defined(CONFIG_MIPS_MT_SMTC) - c->vpe_id = (read_c0_tcbind() >> TCBIND_CURVPE_SHIFT) & TCBIND_CURVPE; -#endif -#ifdef CONFIG_MIPS_MT_SMTC - c->tc_id = (read_c0_tcbind() & TCBIND_CURTC) >> TCBIND_CURTC_SHIFT; -#endif -} - -static void cmp_smp_finish(void) -{ - pr_debug("SMPCMP: CPU%d: %s\n", smp_processor_id(), __func__); - - /* CDFIXME: remove this? */ - write_c0_compare(read_c0_count() + (8 * mips_hpt_frequency / HZ)); - -#ifdef CONFIG_MIPS_MT_FPAFF - /* If we have an FPU, enroll ourselves in the FPU-full mask */ - if (cpu_has_fpu) - cpu_set(smp_processor_id(), mt_fpu_cpumask); -#endif /* CONFIG_MIPS_MT_FPAFF */ - - local_irq_enable(); -} - -static void cmp_cpus_done(void) -{ - pr_debug("SMPCMP: CPU%d: %s\n", smp_processor_id(), __func__); -} - -/* - * Setup the PC, SP, and GP of a secondary processor and start it running - * smp_bootstrap is the place to resume from - * __KSTK_TOS(idle) is apparently the stack pointer - * (unsigned long)idle->thread_info the gp - */ -static void cmp_boot_secondary(int cpu, struct task_struct *idle) -{ - struct thread_info *gp = task_thread_info(idle); - unsigned long sp = __KSTK_TOS(idle); - unsigned long pc = (unsigned long)&smp_bootstrap; - unsigned long a0 = 0; - - pr_debug("SMPCMP: CPU%d: %s cpu %d\n", smp_processor_id(), - __func__, cpu); - -#if 0 - /* Needed? */ - flush_icache_range((unsigned long)gp, - (unsigned long)(gp + sizeof(struct thread_info))); -#endif - - amon_cpu_start(cpu, pc, sp, (unsigned long)gp, a0); -} - -/* - * Common setup before any secondaries are started - */ -void __init cmp_smp_setup(void) -{ - int i; - int ncpu = 0; - - pr_debug("SMPCMP: CPU%d: %s\n", smp_processor_id(), __func__); - -#ifdef CONFIG_MIPS_MT_FPAFF - /* If we have an FPU, enroll ourselves in the FPU-full mask */ - if (cpu_has_fpu) - cpu_set(0, mt_fpu_cpumask); -#endif /* CONFIG_MIPS_MT_FPAFF */ - - for (i = 1; i < NR_CPUS; i++) { - if (amon_cpu_avail(i)) { - set_cpu_possible(i, true); - __cpu_number_map[i] = ++ncpu; - __cpu_logical_map[ncpu] = i; - } - } - - if (cpu_has_mipsmt) { - unsigned int nvpe, mvpconf0 = read_c0_mvpconf0(); - - nvpe = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; - smp_num_siblings = nvpe; - } - pr_info("Detected %i available secondary CPU(s)\n", ncpu); -} - -void __init cmp_prepare_cpus(unsigned int max_cpus) -{ - pr_debug("SMPCMP: CPU%d: %s max_cpus=%d\n", - smp_processor_id(), __func__, max_cpus); - - /* - * FIXME: some of these options are per-system, some per-core and - * some per-cpu - */ - mips_mt_set_cpuoptions(); -} - -struct plat_smp_ops cmp_smp_ops = { - .send_ipi_single = cmp_send_ipi_single, - .send_ipi_mask = cmp_send_ipi_mask, - .init_secondary = cmp_init_secondary, - .smp_finish = cmp_smp_finish, - .cpus_done = cmp_cpus_done, - .boot_secondary = cmp_boot_secondary, - .smp_setup = cmp_smp_setup, - .prepare_cpus = cmp_prepare_cpus, -}; diff --git a/arch/mips/kernel/smp-cps.c b/arch/mips/kernel/smp-cps.c new file mode 100644 index 000000000000..22d4f9ff3ae2 --- /dev/null +++ b/arch/mips/kernel/smp-cps.c @@ -0,0 +1,979 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2013 Imagination Technologies + * Author: Paul Burton <paul.burton@mips.com> + */ + +#include <linux/cpu.h> +#include <linux/delay.h> +#include <linux/io.h> +#include <linux/memblock.h> +#include <linux/sched/task_stack.h> +#include <linux/sched/hotplug.h> +#include <linux/slab.h> +#include <linux/smp.h> +#include <linux/types.h> +#include <linux/irq.h> + +#include <asm/bcache.h> +#include <asm/mips-cps.h> +#include <asm/mips_mt.h> +#include <asm/mipsregs.h> +#include <asm/pm-cps.h> +#include <asm/r4kcache.h> +#include <asm/regdef.h> +#include <asm/smp.h> +#include <asm/smp-cps.h> +#include <asm/time.h> +#include <asm/uasm.h> + +#define BEV_VEC_SIZE 0x500 +#define BEV_VEC_ALIGN 0x1000 + +enum label_id { + label_not_nmi = 1, +}; + +UASM_L_LA(_not_nmi) + +static u64 core_entry_reg; +static phys_addr_t cps_vec_pa; + +struct cluster_boot_config *mips_cps_cluster_bootcfg; + +static void power_up_other_cluster(unsigned int cluster) +{ + u32 stat, seq_state; + unsigned int timeout; + + mips_cm_lock_other(cluster, CM_GCR_Cx_OTHER_CORE_CM, 0, + CM_GCR_Cx_OTHER_BLOCK_LOCAL); + stat = read_cpc_co_stat_conf(); + mips_cm_unlock_other(); + + seq_state = stat & CPC_Cx_STAT_CONF_SEQSTATE; + seq_state >>= __ffs(CPC_Cx_STAT_CONF_SEQSTATE); + if (seq_state == CPC_Cx_STAT_CONF_SEQSTATE_U5) + return; + + /* Set endianness & power up the CM */ + mips_cm_lock_other(cluster, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL); + write_cpc_redir_sys_config(IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)); + write_cpc_redir_pwrup_ctl(1); + mips_cm_unlock_other(); + + /* Wait for the CM to start up */ + timeout = 1000; + mips_cm_lock_other(cluster, CM_GCR_Cx_OTHER_CORE_CM, 0, + CM_GCR_Cx_OTHER_BLOCK_LOCAL); + while (1) { + stat = read_cpc_co_stat_conf(); + seq_state = stat & CPC_Cx_STAT_CONF_SEQSTATE; + seq_state >>= __ffs(CPC_Cx_STAT_CONF_SEQSTATE); + if (seq_state == CPC_Cx_STAT_CONF_SEQSTATE_U5) + break; + + if (timeout) { + mdelay(1); + timeout--; + } else { + pr_warn("Waiting for cluster %u CM to power up... STAT_CONF=0x%x\n", + cluster, stat); + mdelay(1000); + } + } + + mips_cm_unlock_other(); +} + +static unsigned __init core_vpe_count(unsigned int cluster, unsigned core) +{ + return min(smp_max_threads, mips_cps_numvps(cluster, core)); +} + +static void __init *mips_cps_build_core_entry(void *addr) +{ + extern void (*nmi_handler)(void); + u32 *p = addr; + u32 val; + struct uasm_label labels[2]; + struct uasm_reloc relocs[2]; + struct uasm_label *l = labels; + struct uasm_reloc *r = relocs; + + memset(labels, 0, sizeof(labels)); + memset(relocs, 0, sizeof(relocs)); + + uasm_i_mfc0(&p, GPR_K0, C0_STATUS); + UASM_i_LA(&p, GPR_T9, ST0_NMI); + uasm_i_and(&p, GPR_K0, GPR_K0, GPR_T9); + + uasm_il_bnez(&p, &r, GPR_K0, label_not_nmi); + uasm_i_nop(&p); + UASM_i_LA(&p, GPR_K0, (long)&nmi_handler); + + uasm_l_not_nmi(&l, p); + + val = CAUSEF_IV; + uasm_i_lui(&p, GPR_K0, val >> 16); + uasm_i_ori(&p, GPR_K0, GPR_K0, val & 0xffff); + uasm_i_mtc0(&p, GPR_K0, C0_CAUSE); + val = ST0_CU1 | ST0_CU0 | ST0_BEV | ST0_KX_IF_64; + uasm_i_lui(&p, GPR_K0, val >> 16); + uasm_i_ori(&p, GPR_K0, GPR_K0, val & 0xffff); + uasm_i_mtc0(&p, GPR_K0, C0_STATUS); + uasm_i_ehb(&p); + uasm_i_ori(&p, GPR_A0, 0, read_c0_config() & CONF_CM_CMASK); + UASM_i_LA(&p, GPR_A1, (long)mips_gcr_base); +#if defined(KBUILD_64BIT_SYM32) || defined(CONFIG_32BIT) + UASM_i_LA(&p, GPR_T9, CKSEG1ADDR(__pa_symbol(mips_cps_core_boot))); +#else + UASM_i_LA(&p, GPR_T9, TO_UNCAC(__pa_symbol(mips_cps_core_boot))); +#endif + uasm_i_jr(&p, GPR_T9); + uasm_i_nop(&p); + + uasm_resolve_relocs(relocs, labels); + + return p; +} + +static bool __init check_64bit_reset(void) +{ + bool cx_64bit_reset = false; + + mips_cm_lock_other(0, 0, 0, CM_GCR_Cx_OTHER_BLOCK_LOCAL); + write_gcr_co_reset64_base(CM_GCR_Cx_RESET64_BASE_BEVEXCBASE); + if ((read_gcr_co_reset64_base() & CM_GCR_Cx_RESET64_BASE_BEVEXCBASE) == + CM_GCR_Cx_RESET64_BASE_BEVEXCBASE) + cx_64bit_reset = true; + mips_cm_unlock_other(); + + return cx_64bit_reset; +} + +static int __init allocate_cps_vecs(void) +{ + /* Try to allocate in KSEG1 first */ + cps_vec_pa = memblock_phys_alloc_range(BEV_VEC_SIZE, BEV_VEC_ALIGN, + 0x0, CSEGX_SIZE - 1); + + if (cps_vec_pa) + core_entry_reg = CKSEG1ADDR(cps_vec_pa) & + CM_GCR_Cx_RESET_BASE_BEVEXCBASE; + + if (!cps_vec_pa && mips_cm_is64) { + phys_addr_t end; + + if (check_64bit_reset()) { + pr_info("VP Local Reset Exception Base support 47 bits address\n"); + end = MEMBLOCK_ALLOC_ANYWHERE; + } else { + end = SZ_4G - 1; + } + cps_vec_pa = memblock_phys_alloc_range(BEV_VEC_SIZE, BEV_VEC_ALIGN, 0, end); + if (cps_vec_pa) { + if (check_64bit_reset()) + core_entry_reg = (cps_vec_pa & CM_GCR_Cx_RESET64_BASE_BEVEXCBASE) | + CM_GCR_Cx_RESET_BASE_MODE; + else + core_entry_reg = (cps_vec_pa & CM_GCR_Cx_RESET_BASE_BEVEXCBASE) | + CM_GCR_Cx_RESET_BASE_MODE; + } + } + + if (!cps_vec_pa) + return -ENOMEM; + + return 0; +} + +static void __init setup_cps_vecs(void) +{ + void *cps_vec; + + cps_vec = (void *)CKSEG1ADDR_OR_64BIT(cps_vec_pa); + mips_cps_build_core_entry(cps_vec); + + memcpy(cps_vec + 0x200, &excep_tlbfill, 0x80); + memcpy(cps_vec + 0x280, &excep_xtlbfill, 0x80); + memcpy(cps_vec + 0x300, &excep_cache, 0x80); + memcpy(cps_vec + 0x380, &excep_genex, 0x80); + memcpy(cps_vec + 0x400, &excep_intex, 0x80); + memcpy(cps_vec + 0x480, &excep_ejtag, 0x80); + + /* Make sure no prefetched data in cache */ + blast_inv_dcache_range(CKSEG0ADDR_OR_64BIT(cps_vec_pa), CKSEG0ADDR_OR_64BIT(cps_vec_pa) + BEV_VEC_SIZE); + bc_inv(CKSEG0ADDR_OR_64BIT(cps_vec_pa), BEV_VEC_SIZE); + __sync(); +} + +static void __init cps_smp_setup(void) +{ + unsigned int nclusters, ncores, nvpes, core_vpes; + int cl, c, v; + + /* Detect & record VPE topology */ + nvpes = 0; + nclusters = mips_cps_numclusters(); + pr_info("%s topology ", cpu_has_mips_r6 ? "VP" : "VPE"); + for (cl = 0; cl < nclusters; cl++) { + if (cl > 0) + pr_cont(","); + pr_cont("{"); + + if (mips_cm_revision() >= CM_REV_CM3_5) + power_up_other_cluster(cl); + + ncores = mips_cps_numcores(cl); + for (c = 0; c < ncores; c++) { + core_vpes = core_vpe_count(cl, c); + + if (c > 0) + pr_cont(","); + pr_cont("%u", core_vpes); + + /* Use the number of VPEs in cluster 0 core 0 for smp_num_siblings */ + if (!cl && !c) + smp_num_siblings = core_vpes; + cpumask_set_cpu(nvpes, &__cpu_primary_thread_mask); + + for (v = 0; v < min_t(int, core_vpes, NR_CPUS - nvpes); v++) { + cpu_set_cluster(&cpu_data[nvpes + v], cl); + cpu_set_core(&cpu_data[nvpes + v], c); + cpu_set_vpe_id(&cpu_data[nvpes + v], v); + } + + nvpes += core_vpes; + } + + pr_cont("}"); + } + pr_cont(" total %u\n", nvpes); + + /* Indicate present CPUs (CPU being synonymous with VPE) */ + for (v = 0; v < min_t(unsigned, nvpes, NR_CPUS); v++) { + set_cpu_possible(v, true); + set_cpu_present(v, true); + __cpu_number_map[v] = v; + __cpu_logical_map[v] = v; + } + + /* Set a coherent default CCA (CWB) */ + change_c0_config(CONF_CM_CMASK, 0x5); + + /* Initialise core 0 */ + mips_cps_core_init(); + + /* Make core 0 coherent with everything */ + write_gcr_cl_coherence(0xff); + + if (allocate_cps_vecs()) + pr_err("Failed to allocate CPS vectors\n"); + + if (core_entry_reg && mips_cm_revision() >= CM_REV_CM3) + write_gcr_bev_base(core_entry_reg); + +#ifdef CONFIG_MIPS_MT_FPAFF + /* If we have an FPU, enroll ourselves in the FPU-full mask */ + if (cpu_has_fpu) + cpumask_set_cpu(0, &mt_fpu_cpumask); +#endif /* CONFIG_MIPS_MT_FPAFF */ +} + +unsigned long calibrate_delay_is_known(void) +{ + int first_cpu_cluster = 0; + + /* The calibration has to be done on the primary CPU of the cluster */ + if (mips_cps_first_online_in_cluster(&first_cpu_cluster)) + return 0; + + return cpu_data[first_cpu_cluster].udelay_val; +} + +static void __init cps_prepare_cpus(unsigned int max_cpus) +{ + unsigned int nclusters, ncores, core_vpes, nvpe = 0, c, cl, cca; + bool cca_unsuitable, cores_limited; + struct cluster_boot_config *cluster_bootcfg; + struct core_boot_config *core_bootcfg; + + mips_mt_set_cpuoptions(); + + if (!core_entry_reg) { + pr_err("core_entry address unsuitable, disabling smp-cps\n"); + goto err_out; + } + + /* Detect whether the CCA is unsuited to multi-core SMP */ + cca = read_c0_config() & CONF_CM_CMASK; + switch (cca) { + case 0x4: /* CWBE */ + case 0x5: /* CWB */ + /* The CCA is coherent, multi-core is fine */ + cca_unsuitable = false; + break; + + default: + /* CCA is not coherent, multi-core is not usable */ + cca_unsuitable = true; + } + + /* Warn the user if the CCA prevents multi-core */ + cores_limited = false; + if (cca_unsuitable || cpu_has_dc_aliases) { + for_each_present_cpu(c) { + if (cpus_are_siblings(smp_processor_id(), c)) + continue; + + set_cpu_present(c, false); + cores_limited = true; + } + } + if (cores_limited) + pr_warn("Using only one core due to %s%s%s\n", + cca_unsuitable ? "unsuitable CCA" : "", + (cca_unsuitable && cpu_has_dc_aliases) ? " & " : "", + cpu_has_dc_aliases ? "dcache aliasing" : ""); + + setup_cps_vecs(); + + /* Allocate cluster boot configuration structs */ + nclusters = mips_cps_numclusters(); + mips_cps_cluster_bootcfg = kcalloc(nclusters, + sizeof(*mips_cps_cluster_bootcfg), + GFP_KERNEL); + if (!mips_cps_cluster_bootcfg) + goto err_out; + + if (nclusters > 1) + mips_cm_update_property(); + + for (cl = 0; cl < nclusters; cl++) { + /* Allocate core boot configuration structs */ + ncores = mips_cps_numcores(cl); + core_bootcfg = kcalloc(ncores, sizeof(*core_bootcfg), + GFP_KERNEL); + if (!core_bootcfg) + goto err_out; + mips_cps_cluster_bootcfg[cl].core_config = core_bootcfg; + + mips_cps_cluster_bootcfg[cl].core_power = + kcalloc(BITS_TO_LONGS(ncores), sizeof(unsigned long), + GFP_KERNEL); + if (!mips_cps_cluster_bootcfg[cl].core_power) + goto err_out; + + /* Allocate VPE boot configuration structs */ + for (c = 0; c < ncores; c++) { + int v; + core_vpes = core_vpe_count(cl, c); + core_bootcfg[c].vpe_config = kcalloc(core_vpes, + sizeof(*core_bootcfg[c].vpe_config), + GFP_KERNEL); + for (v = 0; v < core_vpes; v++) + cpumask_set_cpu(nvpe++, &mips_cps_cluster_bootcfg[cl].cpumask); + if (!core_bootcfg[c].vpe_config) + goto err_out; + } + } + + /* Mark this CPU as powered up & booted */ + cl = cpu_cluster(¤t_cpu_data); + c = cpu_core(¤t_cpu_data); + cluster_bootcfg = &mips_cps_cluster_bootcfg[cl]; + cpu_smt_set_num_threads(core_vpes, core_vpes); + core_bootcfg = &cluster_bootcfg->core_config[c]; + bitmap_set(cluster_bootcfg->core_power, cpu_core(¤t_cpu_data), 1); + atomic_set(&core_bootcfg->vpe_mask, 1 << cpu_vpe_id(¤t_cpu_data)); + + return; +err_out: + /* Clean up allocations */ + if (mips_cps_cluster_bootcfg) { + for (cl = 0; cl < nclusters; cl++) { + cluster_bootcfg = &mips_cps_cluster_bootcfg[cl]; + ncores = mips_cps_numcores(cl); + for (c = 0; c < ncores; c++) { + core_bootcfg = &cluster_bootcfg->core_config[c]; + kfree(core_bootcfg->vpe_config); + } + kfree(mips_cps_cluster_bootcfg[c].core_config); + } + kfree(mips_cps_cluster_bootcfg); + mips_cps_cluster_bootcfg = NULL; + } + + /* Effectively disable SMP by declaring CPUs not present */ + for_each_possible_cpu(c) { + if (c == 0) + continue; + set_cpu_present(c, false); + } +} + +static void init_cluster_l2(void) +{ + u32 l2_cfg, l2sm_cop, result; + + while (!mips_cm_is_l2_hci_broken) { + l2_cfg = read_gcr_redir_l2_ram_config(); + + /* If HCI is not supported, use the state machine below */ + if (!(l2_cfg & CM_GCR_L2_RAM_CONFIG_PRESENT)) + break; + if (!(l2_cfg & CM_GCR_L2_RAM_CONFIG_HCI_SUPPORTED)) + break; + + /* If the HCI_DONE bit is set, we're finished */ + if (l2_cfg & CM_GCR_L2_RAM_CONFIG_HCI_DONE) + return; + } + + l2sm_cop = read_gcr_redir_l2sm_cop(); + if (WARN(!(l2sm_cop & CM_GCR_L2SM_COP_PRESENT), + "L2 init not supported on this system yet")) + return; + + /* Clear L2 tag registers */ + write_gcr_redir_l2_tag_state(0); + write_gcr_redir_l2_ecc(0); + + /* Ensure the L2 tag writes complete before the state machine starts */ + mb(); + + /* Wait for the L2 state machine to be idle */ + do { + l2sm_cop = read_gcr_redir_l2sm_cop(); + } while (l2sm_cop & CM_GCR_L2SM_COP_RUNNING); + + /* Start a store tag operation */ + l2sm_cop = CM_GCR_L2SM_COP_TYPE_IDX_STORETAG; + l2sm_cop <<= __ffs(CM_GCR_L2SM_COP_TYPE); + l2sm_cop |= CM_GCR_L2SM_COP_CMD_START; + write_gcr_redir_l2sm_cop(l2sm_cop); + + /* Ensure the state machine starts before we poll for completion */ + mb(); + + /* Wait for the operation to be complete */ + do { + l2sm_cop = read_gcr_redir_l2sm_cop(); + result = l2sm_cop & CM_GCR_L2SM_COP_RESULT; + result >>= __ffs(CM_GCR_L2SM_COP_RESULT); + } while (!result); + + WARN(result != CM_GCR_L2SM_COP_RESULT_DONE_OK, + "L2 state machine failed cache init with error %u\n", result); +} + +static void boot_core(unsigned int cluster, unsigned int core, + unsigned int vpe_id) +{ + struct cluster_boot_config *cluster_cfg; + u32 access, stat, seq_state; + unsigned int timeout, ncores; + + cluster_cfg = &mips_cps_cluster_bootcfg[cluster]; + ncores = mips_cps_numcores(cluster); + + if ((cluster != cpu_cluster(¤t_cpu_data)) && + bitmap_empty(cluster_cfg->core_power, ncores)) { + power_up_other_cluster(cluster); + + mips_cm_lock_other(cluster, core, 0, + CM_GCR_Cx_OTHER_BLOCK_GLOBAL); + + /* Ensure cluster GCRs are where we expect */ + write_gcr_redir_base(read_gcr_base()); + write_gcr_redir_cpc_base(read_gcr_cpc_base()); + write_gcr_redir_gic_base(read_gcr_gic_base()); + + init_cluster_l2(); + + /* Mirror L2 configuration */ + write_gcr_redir_l2_only_sync_base(read_gcr_l2_only_sync_base()); + write_gcr_redir_l2_pft_control(read_gcr_l2_pft_control()); + write_gcr_redir_l2_pft_control_b(read_gcr_l2_pft_control_b()); + + /* Mirror ECC/parity setup */ + write_gcr_redir_err_control(read_gcr_err_control()); + + /* Set BEV base */ + write_gcr_redir_bev_base(core_entry_reg); + + mips_cm_unlock_other(); + } + + if (cluster != cpu_cluster(¤t_cpu_data)) { + mips_cm_lock_other(cluster, core, 0, + CM_GCR_Cx_OTHER_BLOCK_GLOBAL); + + /* Ensure the core can access the GCRs */ + access = read_gcr_redir_access(); + access |= BIT(core); + write_gcr_redir_access(access); + + mips_cm_unlock_other(); + } else { + /* Ensure the core can access the GCRs */ + access = read_gcr_access(); + access |= BIT(core); + write_gcr_access(access); + } + + /* Select the appropriate core */ + mips_cm_lock_other(cluster, core, 0, CM_GCR_Cx_OTHER_BLOCK_LOCAL); + + /* Set its reset vector */ + if (mips_cm_is64) + write_gcr_co_reset64_base(core_entry_reg); + else + write_gcr_co_reset_base(core_entry_reg); + + /* Ensure its coherency is disabled */ + write_gcr_co_coherence(0); + + /* Start it with the legacy memory map and exception base */ + write_gcr_co_reset_ext_base(CM_GCR_Cx_RESET_EXT_BASE_UEB); + + /* Ensure the core can access the GCRs */ + if (mips_cm_revision() < CM_REV_CM3) + set_gcr_access(1 << core); + else + set_gcr_access_cm3(1 << core); + + if (mips_cpc_present()) { + /* Reset the core */ + mips_cpc_lock_other(core); + + if (mips_cm_revision() >= CM_REV_CM3) { + /* Run only the requested VP following the reset */ + write_cpc_co_vp_stop(0xf); + write_cpc_co_vp_run(1 << vpe_id); + + /* + * Ensure that the VP_RUN register is written before the + * core leaves reset. + */ + wmb(); + } + + write_cpc_co_cmd(CPC_Cx_CMD_RESET); + + timeout = 100; + while (true) { + stat = read_cpc_co_stat_conf(); + seq_state = stat & CPC_Cx_STAT_CONF_SEQSTATE; + seq_state >>= __ffs(CPC_Cx_STAT_CONF_SEQSTATE); + + /* U6 == coherent execution, ie. the core is up */ + if (seq_state == CPC_Cx_STAT_CONF_SEQSTATE_U6) + break; + + /* Delay a little while before we start warning */ + if (timeout) { + timeout--; + mdelay(10); + continue; + } + + pr_warn("Waiting for core %u to start... STAT_CONF=0x%x\n", + core, stat); + mdelay(1000); + } + + mips_cpc_unlock_other(); + } else { + /* Take the core out of reset */ + write_gcr_co_reset_release(0); + } + + mips_cm_unlock_other(); + + /* The core is now powered up */ + bitmap_set(cluster_cfg->core_power, core, 1); + + /* + * Restore CM_PWRUP=0 so that the CM can power down if all the cores in + * the cluster do (eg. if they're all removed via hotplug. + */ + if (mips_cm_revision() >= CM_REV_CM3_5) { + mips_cm_lock_other(cluster, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL); + write_cpc_redir_pwrup_ctl(0); + mips_cm_unlock_other(); + } +} + +static void remote_vpe_boot(void *dummy) +{ + unsigned int cluster = cpu_cluster(¤t_cpu_data); + unsigned core = cpu_core(¤t_cpu_data); + struct cluster_boot_config *cluster_cfg = + &mips_cps_cluster_bootcfg[cluster]; + struct core_boot_config *core_cfg = &cluster_cfg->core_config[core]; + + mips_cps_boot_vpes(core_cfg, cpu_vpe_id(¤t_cpu_data)); +} + +static int cps_boot_secondary(int cpu, struct task_struct *idle) +{ + unsigned int cluster = cpu_cluster(&cpu_data[cpu]); + unsigned core = cpu_core(&cpu_data[cpu]); + unsigned vpe_id = cpu_vpe_id(&cpu_data[cpu]); + struct cluster_boot_config *cluster_cfg = + &mips_cps_cluster_bootcfg[cluster]; + struct core_boot_config *core_cfg = &cluster_cfg->core_config[core]; + struct vpe_boot_config *vpe_cfg = &core_cfg->vpe_config[vpe_id]; + unsigned int remote; + int err; + + vpe_cfg->pc = (unsigned long)&smp_bootstrap; + vpe_cfg->sp = __KSTK_TOS(idle); + vpe_cfg->gp = (unsigned long)task_thread_info(idle); + + atomic_or(1 << cpu_vpe_id(&cpu_data[cpu]), &core_cfg->vpe_mask); + + preempt_disable(); + + if (!test_bit(core, cluster_cfg->core_power)) { + /* Boot a VPE on a powered down core */ + boot_core(cluster, core, vpe_id); + goto out; + } + + if (cpu_has_vp) { + mips_cm_lock_other(cluster, core, vpe_id, + CM_GCR_Cx_OTHER_BLOCK_LOCAL); + if (mips_cm_is64) + write_gcr_co_reset64_base(core_entry_reg); + else + write_gcr_co_reset_base(core_entry_reg); + mips_cm_unlock_other(); + } + + if (!cpus_are_siblings(cpu, smp_processor_id())) { + /* Boot a VPE on another powered up core */ + for (remote = 0; remote < NR_CPUS; remote++) { + if (!cpus_are_siblings(cpu, remote)) + continue; + if (cpu_online(remote)) + break; + } + if (remote >= NR_CPUS) { + pr_crit("No online CPU in core %u to start CPU%d\n", + core, cpu); + goto out; + } + + err = smp_call_function_single(remote, remote_vpe_boot, + NULL, 1); + if (err) + panic("Failed to call remote CPU\n"); + goto out; + } + + BUG_ON(!cpu_has_mipsmt && !cpu_has_vp); + + /* Boot a VPE on this core */ + mips_cps_boot_vpes(core_cfg, vpe_id); +out: + preempt_enable(); + return 0; +} + +static void cps_init_secondary(void) +{ + int core = cpu_core(¤t_cpu_data); + + /* Disable MT - we only want to run 1 TC per VPE */ + if (cpu_has_mipsmt) + dmt(); + + if (mips_cm_revision() >= CM_REV_CM3) { + unsigned int ident = read_gic_vl_ident(); + + /* + * Ensure that our calculation of the VP ID matches up with + * what the GIC reports, otherwise we'll have configured + * interrupts incorrectly. + */ + BUG_ON(ident != mips_cm_vp_id(smp_processor_id())); + } + + if (core > 0 && !read_gcr_cl_coherence()) + pr_warn("Core %u is not in coherent domain\n", core); + + if (cpu_has_veic) + clear_c0_status(ST0_IM); + else + change_c0_status(ST0_IM, STATUSF_IP2 | STATUSF_IP3 | + STATUSF_IP4 | STATUSF_IP5 | + STATUSF_IP6 | STATUSF_IP7); +} + +static void cps_smp_finish(void) +{ + write_c0_compare(read_c0_count() + (8 * mips_hpt_frequency / HZ)); + +#ifdef CONFIG_MIPS_MT_FPAFF + /* If we have an FPU, enroll ourselves in the FPU-full mask */ + if (cpu_has_fpu) + cpumask_set_cpu(smp_processor_id(), &mt_fpu_cpumask); +#endif /* CONFIG_MIPS_MT_FPAFF */ + + local_irq_enable(); +} + +#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_KEXEC_CORE) + +enum cpu_death { + CPU_DEATH_HALT, + CPU_DEATH_POWER, +}; + +static void cps_shutdown_this_cpu(enum cpu_death death) +{ + unsigned int cpu, core, vpe_id; + + cpu = smp_processor_id(); + core = cpu_core(&cpu_data[cpu]); + + if (death == CPU_DEATH_HALT) { + vpe_id = cpu_vpe_id(&cpu_data[cpu]); + + pr_debug("Halting core %d VP%d\n", core, vpe_id); + if (cpu_has_mipsmt) { + /* Halt this TC */ + write_c0_tchalt(TCHALT_H); + instruction_hazard(); + } else if (cpu_has_vp) { + write_cpc_cl_vp_stop(1 << vpe_id); + + /* Ensure that the VP_STOP register is written */ + wmb(); + } + } else { + if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) { + pr_debug("Gating power to core %d\n", core); + /* Power down the core */ + cps_pm_enter_state(CPS_PM_POWER_GATED); + } + } +} + +#ifdef CONFIG_KEXEC_CORE + +static void cps_kexec_nonboot_cpu(void) +{ + if (cpu_has_mipsmt || cpu_has_vp) + cps_shutdown_this_cpu(CPU_DEATH_HALT); + else + cps_shutdown_this_cpu(CPU_DEATH_POWER); +} + +#endif /* CONFIG_KEXEC_CORE */ + +#endif /* CONFIG_HOTPLUG_CPU || CONFIG_KEXEC_CORE */ + +#ifdef CONFIG_HOTPLUG_CPU + +static int cps_cpu_disable(void) +{ + unsigned cpu = smp_processor_id(); + struct cluster_boot_config *cluster_cfg; + struct core_boot_config *core_cfg; + + if (!cps_pm_support_state(CPS_PM_POWER_GATED)) + return -EINVAL; + + cluster_cfg = &mips_cps_cluster_bootcfg[cpu_cluster(¤t_cpu_data)]; + core_cfg = &cluster_cfg->core_config[cpu_core(¤t_cpu_data)]; + atomic_sub(1 << cpu_vpe_id(¤t_cpu_data), &core_cfg->vpe_mask); + smp_mb__after_atomic(); + set_cpu_online(cpu, false); + calculate_cpu_foreign_map(); + irq_migrate_all_off_this_cpu(); + + return 0; +} + +static unsigned cpu_death_sibling; +static enum cpu_death cpu_death; + +void play_dead(void) +{ + unsigned int cpu; + + local_irq_disable(); + idle_task_exit(); + cpu = smp_processor_id(); + cpu_death = CPU_DEATH_POWER; + + pr_debug("CPU%d going offline\n", cpu); + + if (cpu_has_mipsmt || cpu_has_vp) { + /* Look for another online VPE within the core */ + for_each_online_cpu(cpu_death_sibling) { + if (!cpus_are_siblings(cpu, cpu_death_sibling)) + continue; + + /* + * There is an online VPE within the core. Just halt + * this TC and leave the core alone. + */ + cpu_death = CPU_DEATH_HALT; + break; + } + } + + cpuhp_ap_report_dead(); + + cps_shutdown_this_cpu(cpu_death); + + /* This should never be reached */ + panic("Failed to offline CPU %u", cpu); +} + +static void wait_for_sibling_halt(void *ptr_cpu) +{ + unsigned cpu = (unsigned long)ptr_cpu; + unsigned vpe_id = cpu_vpe_id(&cpu_data[cpu]); + unsigned halted; + unsigned long flags; + + do { + local_irq_save(flags); + settc(vpe_id); + halted = read_tc_c0_tchalt(); + local_irq_restore(flags); + } while (!(halted & TCHALT_H)); +} + +static void cps_cpu_die(unsigned int cpu) { } + +static void cps_cleanup_dead_cpu(unsigned cpu) +{ + unsigned int cluster = cpu_cluster(&cpu_data[cpu]); + unsigned core = cpu_core(&cpu_data[cpu]); + unsigned int vpe_id = cpu_vpe_id(&cpu_data[cpu]); + ktime_t fail_time; + unsigned stat; + int err; + struct cluster_boot_config *cluster_cfg; + + cluster_cfg = &mips_cps_cluster_bootcfg[cluster]; + + /* + * Now wait for the CPU to actually offline. Without doing this that + * offlining may race with one or more of: + * + * - Onlining the CPU again. + * - Powering down the core if another VPE within it is offlined. + * - A sibling VPE entering a non-coherent state. + * + * In the non-MT halt case (ie. infinite loop) the CPU is doing nothing + * with which we could race, so do nothing. + */ + if (cpu_death == CPU_DEATH_POWER) { + /* + * Wait for the core to enter a powered down or clock gated + * state, the latter happening when a JTAG probe is connected + * in which case the CPC will refuse to power down the core. + */ + fail_time = ktime_add_ms(ktime_get(), 2000); + do { + mips_cm_lock_other(0, core, 0, CM_GCR_Cx_OTHER_BLOCK_LOCAL); + mips_cpc_lock_other(core); + stat = read_cpc_co_stat_conf(); + stat &= CPC_Cx_STAT_CONF_SEQSTATE; + stat >>= __ffs(CPC_Cx_STAT_CONF_SEQSTATE); + mips_cpc_unlock_other(); + mips_cm_unlock_other(); + + if (stat == CPC_Cx_STAT_CONF_SEQSTATE_D0 || + stat == CPC_Cx_STAT_CONF_SEQSTATE_D2 || + stat == CPC_Cx_STAT_CONF_SEQSTATE_U2) + break; + + /* + * The core ought to have powered down, but didn't & + * now we don't really know what state it's in. It's + * likely that its _pwr_up pin has been wired to logic + * 1 & it powered back up as soon as we powered it + * down... + * + * The best we can do is warn the user & continue in + * the hope that the core is doing nothing harmful & + * might behave properly if we online it later. + */ + if (WARN(ktime_after(ktime_get(), fail_time), + "CPU%u hasn't powered down, seq. state %u\n", + cpu, stat)) + break; + } while (1); + + /* Indicate the core is powered off */ + bitmap_clear(cluster_cfg->core_power, core, 1); + } else if (cpu_has_mipsmt) { + /* + * Have a CPU with access to the offlined CPUs registers wait + * for its TC to halt. + */ + err = smp_call_function_single(cpu_death_sibling, + wait_for_sibling_halt, + (void *)(unsigned long)cpu, 1); + if (err) + panic("Failed to call remote sibling CPU\n"); + } else if (cpu_has_vp) { + do { + mips_cm_lock_other(0, core, vpe_id, CM_GCR_Cx_OTHER_BLOCK_LOCAL); + stat = read_cpc_co_vp_running(); + mips_cm_unlock_other(); + } while (stat & (1 << vpe_id)); + } +} + +#endif /* CONFIG_HOTPLUG_CPU */ + +static const struct plat_smp_ops cps_smp_ops = { + .smp_setup = cps_smp_setup, + .prepare_cpus = cps_prepare_cpus, + .boot_secondary = cps_boot_secondary, + .init_secondary = cps_init_secondary, + .smp_finish = cps_smp_finish, + .send_ipi_single = mips_smp_send_ipi_single, + .send_ipi_mask = mips_smp_send_ipi_mask, +#ifdef CONFIG_HOTPLUG_CPU + .cpu_disable = cps_cpu_disable, + .cpu_die = cps_cpu_die, + .cleanup_dead_cpu = cps_cleanup_dead_cpu, +#endif +#ifdef CONFIG_KEXEC_CORE + .kexec_nonboot_cpu = cps_kexec_nonboot_cpu, +#endif +}; + +bool mips_cps_smp_in_use(void) +{ + extern const struct plat_smp_ops *mp_ops; + return mp_ops == &cps_smp_ops; +} + +int register_cps_smp_ops(void) +{ + if (!mips_cm_present()) { + pr_warn("MIPS CPS SMP unable to proceed without a CM\n"); + return -ENODEV; + } + + /* check we have a GIC - we need one for IPIs */ + if (!(read_gcr_gic_status() & CM_GCR_GIC_STATUS_EX)) { + pr_warn("MIPS CPS SMP unable to proceed without a GIC\n"); + return -ENODEV; + } + + register_smp_ops(&cps_smp_ops); + return 0; +} diff --git a/arch/mips/kernel/smp-mt.c b/arch/mips/kernel/smp-mt.c index 3e5164c11cac..7729cc733421 100644 --- a/arch/mips/kernel/smp-mt.c +++ b/arch/mips/kernel/smp-mt.c @@ -1,16 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only /* - * This program is free software; you can distribute it and/or modify it - * under the terms of the GNU General Public License (Version 2) as - * published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. * * Copyright (C) 2004, 05, 06 MIPS Technologies, Inc. * Elizabeth Clarke (beth@mips.com) @@ -22,6 +11,7 @@ #include <linux/cpumask.h> #include <linux/interrupt.h> #include <linux/compiler.h> +#include <linux/sched/task_stack.h> #include <linux/smp.h> #include <linux/atomic.h> @@ -34,7 +24,7 @@ #include <asm/mipsregs.h> #include <asm/mipsmtregs.h> #include <asm/mips_mt.h> -#include <asm/gic.h> +#include <asm/mips-cps.h> static void __init smvp_copy_vpe_config(void) { @@ -56,7 +46,8 @@ static void __init smvp_copy_vpe_config(void) static unsigned int __init smvp_vpe_init(unsigned int tc, unsigned int mvpconf0, unsigned int ncpu) { - if (tc > ((mvpconf0 & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT)) + if (tc >= smp_max_threads || + (tc > ((mvpconf0 & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT))) return ncpu; /* Deactivate all but VPE 0 */ @@ -71,6 +62,7 @@ static unsigned int __init smvp_vpe_init(unsigned int tc, unsigned int mvpconf0, /* Record this as available CPU */ set_cpu_possible(tc, true); + set_cpu_present(tc, true); __cpu_number_map[tc] = ++ncpu; __cpu_logical_map[ncpu] = tc; } @@ -81,6 +73,8 @@ static unsigned int __init smvp_vpe_init(unsigned int tc, unsigned int mvpconf0, if (tc != 0) smvp_copy_vpe_config(); + cpu_set_vpe_id(&cpu_data[ncpu], tc); + return ncpu; } @@ -112,57 +106,19 @@ static void __init smvp_tc_init(unsigned int tc, unsigned int mvpconf0) write_tc_c0_tchalt(TCHALT_H); } -static void vsmp_send_ipi_single(int cpu, unsigned int action) +static void vsmp_init_secondary(void) { - int i; - unsigned long flags; - int vpflags; - - local_irq_save(flags); - - vpflags = dvpe(); /* can't access the other CPU's registers whilst MVPE enabled */ - - switch (action) { - case SMP_CALL_FUNCTION: - i = C_SW1; - break; - - case SMP_RESCHEDULE_YOURSELF: - default: - i = C_SW0; - break; - } - - /* 1:1 mapping of vpe and tc... */ - settc(cpu); - write_vpe_c0_cause(read_vpe_c0_cause() | i); - evpe(vpflags); - - local_irq_restore(flags); -} - -static void vsmp_send_ipi_mask(const struct cpumask *mask, unsigned int action) -{ - unsigned int i; - - for_each_cpu(i, mask) - vsmp_send_ipi_single(i, action); -} - -static void __cpuinit vsmp_init_secondary(void) -{ -#ifdef CONFIG_IRQ_GIC /* This is Malta specific: IPI,performance and timer interrupts */ - if (gic_present) - change_c0_status(ST0_IM, STATUSF_IP3 | STATUSF_IP4 | + if (mips_gic_present()) + change_c0_status(ST0_IM, STATUSF_IP2 | STATUSF_IP3 | + STATUSF_IP4 | STATUSF_IP5 | STATUSF_IP6 | STATUSF_IP7); else -#endif change_c0_status(ST0_IM, STATUSF_IP0 | STATUSF_IP1 | STATUSF_IP6 | STATUSF_IP7); } -static void __cpuinit vsmp_smp_finish(void) +static void vsmp_smp_finish(void) { /* CDFIXME: remove this? */ write_c0_compare(read_c0_count() + (8* mips_hpt_frequency/HZ)); @@ -170,16 +126,12 @@ static void __cpuinit vsmp_smp_finish(void) #ifdef CONFIG_MIPS_MT_FPAFF /* If we have an FPU, enroll ourselves in the FPU-full mask */ if (cpu_has_fpu) - cpu_set(smp_processor_id(), mt_fpu_cpumask); + cpumask_set_cpu(smp_processor_id(), &mt_fpu_cpumask); #endif /* CONFIG_MIPS_MT_FPAFF */ local_irq_enable(); } -static void vsmp_cpus_done(void) -{ -} - /* * Setup the PC, SP, and GP of a secondary processor and start it * running! @@ -188,7 +140,7 @@ static void vsmp_cpus_done(void) * (unsigned long)idle->thread_info the gp * assumes a 1:1 mapping of TC => VPE */ -static void __cpuinit vsmp_boot_secondary(int cpu, struct task_struct *idle) +static int vsmp_boot_secondary(int cpu, struct task_struct *idle) { struct thread_info *gp = task_thread_info(idle); dvpe(); @@ -220,6 +172,8 @@ static void __cpuinit vsmp_boot_secondary(int cpu, struct task_struct *idle) clear_c0_mvpcontrol(MVPCONTROL_VPC); evpe(EVPE_ENABLE); + + return 0; } /* @@ -235,7 +189,7 @@ static void __init vsmp_smp_setup(void) #ifdef CONFIG_MIPS_MT_FPAFF /* If we have an FPU, enroll ourselves in the FPU-full mask */ if (cpu_has_fpu) - cpu_set(0, mt_fpu_cpumask); + cpumask_set_cpu(0, &mt_fpu_cpumask); #endif /* CONFIG_MIPS_MT_FPAFF */ if (!cpu_has_mipsmt) return; @@ -275,13 +229,13 @@ static void __init vsmp_prepare_cpus(unsigned int max_cpus) mips_mt_set_cpuoptions(); } -struct plat_smp_ops vsmp_smp_ops = { - .send_ipi_single = vsmp_send_ipi_single, - .send_ipi_mask = vsmp_send_ipi_mask, +const struct plat_smp_ops vsmp_smp_ops = { + .send_ipi_single = mips_smp_send_ipi_single, + .send_ipi_mask = mips_smp_send_ipi_mask, .init_secondary = vsmp_init_secondary, .smp_finish = vsmp_smp_finish, - .cpus_done = vsmp_cpus_done, .boot_secondary = vsmp_boot_secondary, .smp_setup = vsmp_smp_setup, .prepare_cpus = vsmp_prepare_cpus, }; + diff --git a/arch/mips/kernel/smp-up.c b/arch/mips/kernel/smp-up.c index 00500fea2750..525d3196f793 100644 --- a/arch/mips/kernel/smp-up.c +++ b/arch/mips/kernel/smp-up.c @@ -28,24 +28,20 @@ static inline void up_send_ipi_mask(const struct cpumask *mask, * After we've done initial boot, this function is called to allow the * board code to clean up state, if needed */ -static void __cpuinit up_init_secondary(void) +static void up_init_secondary(void) { } -static void __cpuinit up_smp_finish(void) -{ -} - -/* Hook for after all CPUs are online */ -static void up_cpus_done(void) +static void up_smp_finish(void) { } /* * Firmware CPU startup hook */ -static void __cpuinit up_boot_secondary(int cpu, struct task_struct *idle) +static int up_boot_secondary(int cpu, struct task_struct *idle) { + return 0; } static void __init up_smp_setup(void) @@ -68,12 +64,11 @@ static void up_cpu_die(unsigned int cpu) } #endif -struct plat_smp_ops up_smp_ops = { +const struct plat_smp_ops up_smp_ops = { .send_ipi_single = up_send_ipi_single, .send_ipi_mask = up_send_ipi_mask, .init_secondary = up_init_secondary, .smp_finish = up_smp_finish, - .cpus_done = up_cpus_done, .boot_secondary = up_boot_secondary, .smp_setup = up_smp_setup, .prepare_cpus = up_prepare_cpus, diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index 6e7862ab46cc..4868e79f3b30 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -1,17 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright (C) 2000, 2001 Kanoj Sarcar * Copyright (C) 2000, 2001 Ralf Baechle @@ -22,34 +10,35 @@ #include <linux/delay.h> #include <linux/init.h> #include <linux/interrupt.h> +#include <linux/profile.h> #include <linux/smp.h> #include <linux/spinlock.h> #include <linux/threads.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/time.h> #include <linux/timex.h> -#include <linux/sched.h> +#include <linux/sched/mm.h> #include <linux/cpumask.h> #include <linux/cpu.h> #include <linux/err.h> #include <linux/ftrace.h> +#include <linux/irqdomain.h> +#include <linux/of.h> +#include <linux/of_irq.h> #include <linux/atomic.h> #include <asm/cpu.h> +#include <asm/ginvt.h> #include <asm/processor.h> #include <asm/idle.h> #include <asm/r4k-timer.h> +#include <asm/mips-cps.h> #include <asm/mmu_context.h> #include <asm/time.h> #include <asm/setup.h> +#include <asm/maar.h> -#ifdef CONFIG_MIPS_MT_SMTC -#include <asm/mipsmtregs.h> -#endif /* CONFIG_MIPS_MT_SMTC */ - -volatile cpumask_t cpu_callin_map; /* Bitmask of started secondaries */ - -int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ +int __cpu_number_map[CONFIG_MIPS_NR_CPU_NR_MAP]; /* Map physical to logical */ EXPORT_SYMBOL(__cpu_number_map); int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */ @@ -63,30 +52,115 @@ EXPORT_SYMBOL(smp_num_siblings); cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly; EXPORT_SYMBOL(cpu_sibling_map); +/* representing the core map of multi-core chips of each logical CPU */ +cpumask_t cpu_core_map[NR_CPUS] __read_mostly; +EXPORT_SYMBOL(cpu_core_map); + +#ifndef CONFIG_HOTPLUG_PARALLEL +static DECLARE_COMPLETION(cpu_starting); +static DECLARE_COMPLETION(cpu_running); +#endif + +/* + * A logical cpu mask containing only one VPE per core to + * reduce the number of IPIs on large MT systems. + */ +cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly; +EXPORT_SYMBOL(cpu_foreign_map); + /* representing cpus for which sibling maps can be computed */ static cpumask_t cpu_sibling_setup_map; +/* representing cpus for which core maps can be computed */ +static cpumask_t cpu_core_setup_map; + +cpumask_t cpu_coherent_mask; + +struct cpumask __cpu_primary_thread_mask __read_mostly; + +unsigned int smp_max_threads __initdata = UINT_MAX; + +static int __init early_nosmt(char *s) +{ + smp_max_threads = 1; + return 0; +} +early_param("nosmt", early_nosmt); + +static int __init early_smt(char *s) +{ + get_option(&s, &smp_max_threads); + /* Ensure at least one thread is available */ + smp_max_threads = clamp_val(smp_max_threads, 1U, UINT_MAX); + return 0; +} +early_param("smt", early_smt); + +#ifdef CONFIG_GENERIC_IRQ_IPI +static struct irq_desc *call_desc; +static struct irq_desc *sched_desc; +#endif + static inline void set_cpu_sibling_map(int cpu) { int i; - cpu_set(cpu, cpu_sibling_setup_map); + cpumask_set_cpu(cpu, &cpu_sibling_setup_map); if (smp_num_siblings > 1) { - for_each_cpu_mask(i, cpu_sibling_setup_map) { - if (cpu_data[cpu].core == cpu_data[i].core) { - cpu_set(i, cpu_sibling_map[cpu]); - cpu_set(cpu, cpu_sibling_map[i]); + for_each_cpu(i, &cpu_sibling_setup_map) { + if (cpus_are_siblings(cpu, i)) { + cpumask_set_cpu(i, &cpu_sibling_map[cpu]); + cpumask_set_cpu(cpu, &cpu_sibling_map[i]); } } } else - cpu_set(cpu, cpu_sibling_map[cpu]); + cpumask_set_cpu(cpu, &cpu_sibling_map[cpu]); +} + +static inline void set_cpu_core_map(int cpu) +{ + int i; + + cpumask_set_cpu(cpu, &cpu_core_setup_map); + + for_each_cpu(i, &cpu_core_setup_map) { + if (cpu_data[cpu].package == cpu_data[i].package) { + cpumask_set_cpu(i, &cpu_core_map[cpu]); + cpumask_set_cpu(cpu, &cpu_core_map[i]); + } + } } -struct plat_smp_ops *mp_ops; +/* + * Calculate a new cpu_foreign_map mask whenever a + * new cpu appears or disappears. + */ +void calculate_cpu_foreign_map(void) +{ + int i, k, core_present; + cpumask_t temp_foreign_map; + + /* Re-calculate the mask */ + cpumask_clear(&temp_foreign_map); + for_each_online_cpu(i) { + core_present = 0; + for_each_cpu(k, &temp_foreign_map) + if (cpus_are_siblings(i, k)) + core_present = 1; + if (!core_present) + cpumask_set_cpu(i, &temp_foreign_map); + } + + for_each_online_cpu(i) + cpumask_andnot(&cpu_foreign_map[i], + &temp_foreign_map, &cpu_sibling_map[i]); +} + +const struct plat_smp_ops *mp_ops; EXPORT_SYMBOL(mp_ops); -__cpuinit void register_smp_ops(struct plat_smp_ops *ops) +void register_smp_ops(const struct plat_smp_ops *ops) { if (mp_ops) printk(KERN_WARNING "Overriding previously set SMP ops\n"); @@ -94,45 +168,240 @@ __cpuinit void register_smp_ops(struct plat_smp_ops *ops) mp_ops = ops; } +#ifdef CONFIG_GENERIC_IRQ_IPI +void mips_smp_send_ipi_single(int cpu, unsigned int action) +{ + mips_smp_send_ipi_mask(cpumask_of(cpu), action); +} + +void mips_smp_send_ipi_mask(const struct cpumask *mask, unsigned int action) +{ + unsigned long flags; + unsigned int core; + int cpu; + + local_irq_save(flags); + + switch (action) { + case SMP_CALL_FUNCTION: + __ipi_send_mask(call_desc, mask); + break; + + case SMP_RESCHEDULE_YOURSELF: + __ipi_send_mask(sched_desc, mask); + break; + + default: + BUG(); + } + + if (mips_cpc_present()) { + for_each_cpu(cpu, mask) { + if (cpus_are_siblings(cpu, smp_processor_id())) + continue; + + core = cpu_core(&cpu_data[cpu]); + + while (!cpumask_test_cpu(cpu, &cpu_coherent_mask)) { + mips_cm_lock_other_cpu(cpu, CM_GCR_Cx_OTHER_BLOCK_LOCAL); + mips_cpc_lock_other(core); + write_cpc_co_cmd(CPC_Cx_CMD_PWRUP); + mips_cpc_unlock_other(); + mips_cm_unlock_other(); + } + } + } + + local_irq_restore(flags); +} + + +static irqreturn_t ipi_resched_interrupt(int irq, void *dev_id) +{ + scheduler_ipi(); + + return IRQ_HANDLED; +} + +static irqreturn_t ipi_call_interrupt(int irq, void *dev_id) +{ + generic_smp_call_function_interrupt(); + + return IRQ_HANDLED; +} + +static void smp_ipi_init_one(unsigned int virq, const char *name, + irq_handler_t handler) +{ + int ret; + + irq_set_handler(virq, handle_percpu_irq); + ret = request_irq(virq, handler, IRQF_PERCPU, name, NULL); + BUG_ON(ret); +} + +static unsigned int call_virq, sched_virq; + +int mips_smp_ipi_allocate(const struct cpumask *mask) +{ + int virq; + struct irq_domain *ipidomain; + struct device_node *node; + + node = of_irq_find_parent(of_root); + ipidomain = irq_find_matching_host(node, DOMAIN_BUS_IPI); + + /* + * Some platforms have half DT setup. So if we found irq node but + * didn't find an ipidomain, try to search for one that is not in the + * DT. + */ + if (node && !ipidomain) + ipidomain = irq_find_matching_host(NULL, DOMAIN_BUS_IPI); + + /* + * There are systems which use IPI IRQ domains, but only have one + * registered when some runtime condition is met. For example a Malta + * kernel may include support for GIC & CPU interrupt controller IPI + * IRQ domains, but if run on a system with no GIC & no MT ASE then + * neither will be supported or registered. + * + * We only have a problem if we're actually using multiple CPUs so fail + * loudly if that is the case. Otherwise simply return, skipping IPI + * setup, if we're running with only a single CPU. + */ + if (!ipidomain) { + BUG_ON(num_present_cpus() > 1); + return 0; + } + + virq = irq_reserve_ipi(ipidomain, mask); + BUG_ON(!virq); + if (!call_virq) + call_virq = virq; + + virq = irq_reserve_ipi(ipidomain, mask); + BUG_ON(!virq); + if (!sched_virq) + sched_virq = virq; + + if (irq_domain_is_ipi_per_cpu(ipidomain)) { + int cpu; + + for_each_cpu(cpu, mask) { + smp_ipi_init_one(call_virq + cpu, "IPI call", + ipi_call_interrupt); + smp_ipi_init_one(sched_virq + cpu, "IPI resched", + ipi_resched_interrupt); + } + } else { + smp_ipi_init_one(call_virq, "IPI call", ipi_call_interrupt); + smp_ipi_init_one(sched_virq, "IPI resched", + ipi_resched_interrupt); + } + + return 0; +} + +int mips_smp_ipi_free(const struct cpumask *mask) +{ + struct irq_domain *ipidomain; + struct device_node *node; + + node = of_irq_find_parent(of_root); + ipidomain = irq_find_matching_host(node, DOMAIN_BUS_IPI); + + /* + * Some platforms have half DT setup. So if we found irq node but + * didn't find an ipidomain, try to search for one that is not in the + * DT. + */ + if (node && !ipidomain) + ipidomain = irq_find_matching_host(NULL, DOMAIN_BUS_IPI); + + BUG_ON(!ipidomain); + + if (irq_domain_is_ipi_per_cpu(ipidomain)) { + int cpu; + + for_each_cpu(cpu, mask) { + free_irq(call_virq + cpu, NULL); + free_irq(sched_virq + cpu, NULL); + } + } + irq_destroy_ipi(call_virq, mask); + irq_destroy_ipi(sched_virq, mask); + return 0; +} + + +static int __init mips_smp_ipi_init(void) +{ + if (num_possible_cpus() == 1) + return 0; + + mips_smp_ipi_allocate(cpu_possible_mask); + + call_desc = irq_to_desc(call_virq); + sched_desc = irq_to_desc(sched_virq); + + return 0; +} +early_initcall(mips_smp_ipi_init); +#endif + /* * First C code run on the secondary CPUs after being started up by * the master. */ -asmlinkage __cpuinit void start_secondary(void) +asmlinkage void start_secondary(void) { - unsigned int cpu; + unsigned int cpu = raw_smp_processor_id(); -#ifdef CONFIG_MIPS_MT_SMTC - /* Only do cpu_probe for first TC of CPU */ - if ((read_c0_tcbind() & TCBIND_CURTC) != 0) - __cpu_name[smp_processor_id()] = __cpu_name[0]; - else -#endif /* CONFIG_MIPS_MT_SMTC */ cpu_probe(); - cpu_report(); per_cpu_trap_init(false); + rcutree_report_cpu_starting(cpu); mips_clockevent_init(); mp_ops->init_secondary(); + cpu_report(); + maar_init(); /* * XXX parity protection should be folded in here when it's converted * to an option instead of something based on .cputype */ +#ifdef CONFIG_HOTPLUG_PARALLEL + cpuhp_ap_sync_alive(); +#endif calibrate_delay(); - preempt_disable(); - cpu = smp_processor_id(); cpu_data[cpu].udelay_val = loops_per_jiffy; + set_cpu_sibling_map(cpu); + set_cpu_core_map(cpu); + + cpumask_set_cpu(cpu, &cpu_coherent_mask); notify_cpu_starting(cpu); - set_cpu_online(cpu, true); +#ifndef CONFIG_HOTPLUG_PARALLEL + /* Notify boot CPU that we're starting & ready to sync counters */ + complete(&cpu_starting); +#endif - set_cpu_sibling_map(cpu); + synchronise_count_slave(cpu); - cpu_set(cpu, cpu_callin_map); + /* The CPU is running and counters synchronised, now mark it online */ + set_cpu_online(cpu, true); - synchronise_count_slave(cpu); + calculate_cpu_foreign_map(); + +#ifndef CONFIG_HOTPLUG_PARALLEL + /* + * Notify boot CPU that we're up & online and it can safely return + * from __cpu_up + */ + complete(&cpu_running); +#endif /* * irq will be enabled in ->smp_finish(), enabling it too early @@ -141,18 +410,7 @@ asmlinkage __cpuinit void start_secondary(void) WARN_ON_ONCE(!irqs_disabled()); mp_ops->smp_finish(); - cpu_startup_entry(CPUHP_ONLINE); -} - -/* - * Call into both interrupt handlers, as we share the IPI for them - */ -void __irq_entry smp_call_function_interrupt(void) -{ - irq_enter(); - generic_smp_call_function_single_interrupt(); - generic_smp_call_function_interrupt(); - irq_exit(); + cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); } static void stop_this_cpu(void *dummy) @@ -160,11 +418,11 @@ static void stop_this_cpu(void *dummy) /* * Remove this CPU: */ + set_cpu_online(smp_processor_id(), false); - for (;;) { - if (cpu_wait) - (*cpu_wait)(); /* Wait if available. */ - } + calculate_cpu_foreign_map(); + local_irq_disable(); + while (1); } void smp_send_stop(void) @@ -174,7 +432,6 @@ void smp_send_stop(void) void __init smp_cpus_done(unsigned int max_cpus) { - mp_ops->cpus_done(); } /* called from main before smp_init() */ @@ -184,38 +441,57 @@ void __init smp_prepare_cpus(unsigned int max_cpus) current_thread_info()->cpu = 0; mp_ops->prepare_cpus(max_cpus); set_cpu_sibling_map(0); + set_cpu_core_map(0); + calculate_cpu_foreign_map(); #ifndef CONFIG_HOTPLUG_CPU init_cpu_present(cpu_possible_mask); #endif + cpumask_copy(&cpu_coherent_mask, cpu_possible_mask); } /* preload SMP state for boot cpu */ -void smp_prepare_boot_cpu(void) +void __init smp_prepare_boot_cpu(void) { + if (mp_ops->prepare_boot_cpu) + mp_ops->prepare_boot_cpu(); set_cpu_possible(0, true); set_cpu_online(0, true); - cpu_set(0, cpu_callin_map); } -int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) +#ifdef CONFIG_HOTPLUG_PARALLEL +int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle) +{ + return mp_ops->boot_secondary(cpu, tidle); +} +#else +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { - mp_ops->boot_secondary(cpu, tidle); + int err; - /* - * Trust is futile. We should really have timeouts ... - */ - while (!cpu_isset(cpu, cpu_callin_map)) - udelay(100); + err = mp_ops->boot_secondary(cpu, tidle); + if (err) + return err; - synchronise_count_master(cpu); + /* Wait for CPU to start and be ready to sync counters */ + if (!wait_for_completion_timeout(&cpu_starting, + msecs_to_jiffies(1000))) { + pr_crit("CPU%u: failed to start\n", cpu); + return -EIO; + } + + /* Wait for CPU to finish startup & mark itself online before return */ + wait_for_completion(&cpu_running); return 0; } +#endif +#ifdef CONFIG_PROFILING /* Not really SMP stuff ... */ int setup_profiling_timer(unsigned int multiplier) { return 0; } +#endif static void flush_tlb_all_ipi(void *info) { @@ -224,12 +500,21 @@ static void flush_tlb_all_ipi(void *info) void flush_tlb_all(void) { + if (cpu_has_mmid) { + htw_stop(); + ginvt_full(); + sync_ginv(); + instruction_hazard(); + htw_start(); + return; + } + on_each_cpu(flush_tlb_all_ipi, NULL, 1); } static void flush_tlb_mm_ipi(void *mm) { - local_flush_tlb_mm((struct mm_struct *)mm); + drop_mmu_context((struct mm_struct *)mm); } /* @@ -239,13 +524,10 @@ static void flush_tlb_mm_ipi(void *mm) * o collapses to normal function call on UP kernels * o collapses to normal function call on systems with a single shared * primary cache. - * o CONFIG_MIPS_MT_SMTC currently implies there is only one physical core. */ static inline void smp_on_other_tlbs(void (*func) (void *info), void *info) { -#ifndef CONFIG_MIPS_MT_SMTC smp_call_function(func, info, 1); -#endif } static inline void smp_on_each_tlb(void (*func) (void *info), void *info) @@ -264,8 +546,8 @@ static inline void smp_on_each_tlb(void (*func) (void *info), void *info) * address spaces, a new context is obtained on the current cpu, and tlb * context on other cpus are invalidated to force a new context allocation * at switch_mm time, should the mm ever be used on other cpus. For - * multithreaded address spaces, intercpu interrupts have to be sent. - * Another case where intercpu interrupts are required is when the target + * multithreaded address spaces, inter-CPU interrupts have to be sent. + * Another case where inter-CPU interrupts are required is when the target * mm might be active on another cpu (eg debuggers doing the flushes on * behalf of debugees, kswapd stealing pages from another process etc). * Kanoj 07/00. @@ -273,19 +555,30 @@ static inline void smp_on_each_tlb(void (*func) (void *info), void *info) void flush_tlb_mm(struct mm_struct *mm) { + if (!mm) + return; + + if (atomic_read(&mm->mm_users) == 0) + return; /* happens as a result of exit_mmap() */ + preempt_disable(); - if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { + if (cpu_has_mmid) { + /* + * No need to worry about other CPUs - the ginvt in + * drop_mmu_context() will be globalized. + */ + } else if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { smp_on_other_tlbs(flush_tlb_mm_ipi, mm); } else { unsigned int cpu; for_each_online_cpu(cpu) { if (cpu != smp_processor_id() && cpu_context(cpu, mm)) - cpu_context(cpu, mm) = 0; + set_cpu_context(cpu, mm, 0); } } - local_flush_tlb_mm(mm); + drop_mmu_context(mm); preempt_enable(); } @@ -306,9 +599,26 @@ static void flush_tlb_range_ipi(void *info) void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { struct mm_struct *mm = vma->vm_mm; + unsigned long addr; + u32 old_mmid; preempt_disable(); - if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { + if (cpu_has_mmid) { + htw_stop(); + old_mmid = read_c0_memorymapid(); + write_c0_memorymapid(cpu_asid(0, mm)); + mtc0_tlbw_hazard(); + addr = round_down(start, PAGE_SIZE * 2); + end = round_up(end, PAGE_SIZE * 2); + do { + ginvt_va_mmid(addr); + sync_ginv(); + addr += PAGE_SIZE * 2; + } while (addr < end); + write_c0_memorymapid(old_mmid); + instruction_hazard(); + htw_start(); + } else if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { struct flush_tlb_data fd = { .vma = vma, .addr1 = start, @@ -316,15 +626,23 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned l }; smp_on_other_tlbs(flush_tlb_range_ipi, &fd); + local_flush_tlb_range(vma, start, end); } else { unsigned int cpu; + int exec = vma->vm_flags & VM_EXEC; for_each_online_cpu(cpu) { + /* + * flush_cache_range() will only fully flush icache if + * the VMA is executable, otherwise we must invalidate + * ASID without it appearing to has_valid_asid() as if + * mm has been completely unused by that CPU. + */ if (cpu != smp_processor_id() && cpu_context(cpu, mm)) - cpu_context(cpu, mm) = 0; + set_cpu_context(cpu, mm, !exec); } + local_flush_tlb_range(vma, start, end); } - local_flush_tlb_range(vma, start, end); preempt_enable(); } @@ -354,23 +672,43 @@ static void flush_tlb_page_ipi(void *info) void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) { + u32 old_mmid; + preempt_disable(); - if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { + if (cpu_has_mmid) { + htw_stop(); + old_mmid = read_c0_memorymapid(); + write_c0_memorymapid(cpu_asid(0, vma->vm_mm)); + mtc0_tlbw_hazard(); + ginvt_va_mmid(page); + sync_ginv(); + write_c0_memorymapid(old_mmid); + instruction_hazard(); + htw_start(); + } else if ((atomic_read(&vma->vm_mm->mm_users) != 1) || + (current->mm != vma->vm_mm)) { struct flush_tlb_data fd = { .vma = vma, .addr1 = page, }; smp_on_other_tlbs(flush_tlb_page_ipi, &fd); + local_flush_tlb_page(vma, page); } else { unsigned int cpu; for_each_online_cpu(cpu) { + /* + * flush_cache_page() only does partial flushes, so + * invalidate ASID without it appearing to + * has_valid_asid() as if mm has been completely unused + * by that CPU. + */ if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm)) - cpu_context(cpu, vma->vm_mm) = 0; + set_cpu_context(cpu, vma->vm_mm, 1); } + local_flush_tlb_page(vma, page); } - local_flush_tlb_page(vma, page); preempt_enable(); } @@ -389,19 +727,33 @@ void flush_tlb_one(unsigned long vaddr) EXPORT_SYMBOL(flush_tlb_page); EXPORT_SYMBOL(flush_tlb_one); -#if defined(CONFIG_KEXEC) -void (*dump_ipi_function_ptr)(void *) = NULL; -void dump_send_ipi(void (*dump_ipi_callback)(void *)) +#ifdef CONFIG_HOTPLUG_CORE_SYNC_DEAD +void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) { - int i; - int cpu = smp_processor_id(); + if (mp_ops->cleanup_dead_cpu) + mp_ops->cleanup_dead_cpu(cpu); +} +#endif - dump_ipi_function_ptr = dump_ipi_callback; - smp_mb(); - for_each_online_cpu(i) - if (i != cpu) - mp_ops->send_ipi_single(i, SMP_DUMP); +#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST +static void tick_broadcast_callee(void *info) +{ + tick_receive_broadcast(); } -EXPORT_SYMBOL(dump_send_ipi); -#endif + +static DEFINE_PER_CPU(call_single_data_t, tick_broadcast_csd) = + CSD_INIT(tick_broadcast_callee, NULL); + +void tick_broadcast(const struct cpumask *mask) +{ + call_single_data_t *csd; + int cpu; + + for_each_cpu(cpu, mask) { + csd = &per_cpu(tick_broadcast_csd, cpu); + smp_call_function_single_async(cpu, csd); + } +} + +#endif /* CONFIG_GENERIC_CLOCKEVENTS_BROADCAST */ diff --git a/arch/mips/kernel/smtc-asm.S b/arch/mips/kernel/smtc-asm.S deleted file mode 100644 index 2866863a39df..000000000000 --- a/arch/mips/kernel/smtc-asm.S +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Assembly Language Functions for MIPS MT SMTC support - */ - -/* - * This file should be built into the kernel only if CONFIG_MIPS_MT_SMTC is set. */ - -#include <asm/regdef.h> -#include <asm/asmmacro.h> -#include <asm/stackframe.h> -#include <asm/irqflags.h> - -/* - * "Software Interrupt" linkage. - * - * This is invoked when an "Interrupt" is sent from one TC to another, - * where the TC to be interrupted is halted, has it's Restart address - * and Status values saved by the "remote control" thread, then modified - * to cause execution to begin here, in kenel mode. This code then - * disguises the TC state as that of an exception and transfers - * control to the general exception or vectored interrupt handler. - */ - .set noreorder - -/* -The __smtc_ipi_vector would use k0 and k1 as temporaries and -1) Set EXL (this is per-VPE, so this can't be done by proxy!) -2) Restore the K/CU and IXMT bits to the pre "exception" state - (EXL means no interrupts and access to the kernel map). -3) Set EPC to be the saved value of TCRestart. -4) Jump to the exception handler entry point passed by the sender. - -CAN WE PROVE THAT WE WON'T DO THIS IF INTS DISABLED?? -*/ - -/* - * Reviled and slandered vision: Set EXL and restore K/CU/IXMT - * state of pre-halt thread, then save everything and call - * thought some function pointer to imaginary_exception, which - * will parse a register value or memory message queue to - * deliver things like interprocessor interrupts. On return - * from that function, jump to the global ret_from_irq code - * to invoke the scheduler and return as appropriate. - */ - -#define PT_PADSLOT4 (PT_R0-8) -#define PT_PADSLOT5 (PT_R0-4) - - .text - .align 5 -FEXPORT(__smtc_ipi_vector) -#ifdef CONFIG_CPU_MICROMIPS - nop -#endif - .set noat - /* Disable thread scheduling to make Status update atomic */ - DMT 27 # dmt k1 - _ehb - /* Set EXL */ - mfc0 k0,CP0_STATUS - ori k0,k0,ST0_EXL - mtc0 k0,CP0_STATUS - _ehb - /* Thread scheduling now inhibited by EXL. Restore TE state. */ - andi k1,k1,VPECONTROL_TE - beqz k1,1f - emt -1: - /* - * The IPI sender has put some information on the anticipated - * kernel stack frame. If we were in user mode, this will be - * built above the saved kernel SP. If we were already in the - * kernel, it will be built above the current CPU SP. - * - * Were we in kernel mode, as indicated by CU0? - */ - sll k1,k0,3 - .set noreorder - bltz k1,2f - move k1,sp - .set reorder - /* - * If previously in user mode, set CU0 and use kernel stack. - */ - li k1,ST0_CU0 - or k1,k1,k0 - mtc0 k1,CP0_STATUS - _ehb - get_saved_sp - /* Interrupting TC will have pre-set values in slots in the new frame */ -2: subu k1,k1,PT_SIZE - /* Load TCStatus Value */ - lw k0,PT_TCSTATUS(k1) - /* Write it to TCStatus to restore CU/KSU/IXMT state */ - mtc0 k0,$2,1 - _ehb - lw k0,PT_EPC(k1) - mtc0 k0,CP0_EPC - /* Save all will redundantly recompute the SP, but use it for now */ - SAVE_ALL - CLI - TRACE_IRQS_OFF - /* Function to be invoked passed stack pad slot 5 */ - lw t0,PT_PADSLOT5(sp) - /* Argument from sender passed in stack pad slot 4 */ - lw a0,PT_PADSLOT4(sp) - LONG_L s0, TI_REGS($28) - LONG_S sp, TI_REGS($28) - PTR_LA ra, ret_from_irq - jr t0 - -/* - * Called from idle loop to provoke processing of queued IPIs - * First IPI message in queue passed as argument. - */ - -LEAF(self_ipi) - /* Before anything else, block interrupts */ - mfc0 t0,CP0_TCSTATUS - ori t1,t0,TCSTATUS_IXMT - mtc0 t1,CP0_TCSTATUS - _ehb - /* We know we're in kernel mode, so prepare stack frame */ - subu t1,sp,PT_SIZE - sw ra,PT_EPC(t1) - sw a0,PT_PADSLOT4(t1) - la t2,ipi_decode - sw t2,PT_PADSLOT5(t1) - /* Save pre-disable value of TCStatus */ - sw t0,PT_TCSTATUS(t1) - j __smtc_ipi_vector - nop -END(self_ipi) diff --git a/arch/mips/kernel/smtc-proc.c b/arch/mips/kernel/smtc-proc.c deleted file mode 100644 index c10aa84c9fa9..000000000000 --- a/arch/mips/kernel/smtc-proc.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * /proc hooks for SMTC kernel - * Copyright (C) 2005 Mips Technologies, Inc - */ - -#include <linux/kernel.h> -#include <linux/sched.h> -#include <linux/cpumask.h> -#include <linux/interrupt.h> - -#include <asm/cpu.h> -#include <asm/processor.h> -#include <linux/atomic.h> -#include <asm/hardirq.h> -#include <asm/mmu_context.h> -#include <asm/mipsregs.h> -#include <asm/cacheflush.h> -#include <linux/proc_fs.h> -#include <linux/seq_file.h> - -#include <asm/smtc_proc.h> - -/* - * /proc diagnostic and statistics hooks - */ - -/* - * Statistics gathered - */ -unsigned long selfipis[NR_CPUS]; - -struct smtc_cpu_proc smtc_cpu_stats[NR_CPUS]; - -atomic_t smtc_fpu_recoveries; - -static int smtc_proc_show(struct seq_file *m, void *v) -{ - int i; - extern unsigned long ebase; - - seq_printf(m, "SMTC Status Word: 0x%08x\n", smtc_status); - seq_printf(m, "Config7: 0x%08x\n", read_c0_config7()); - seq_printf(m, "EBASE: 0x%08lx\n", ebase); - seq_printf(m, "Counter Interrupts taken per CPU (TC)\n"); - for (i=0; i < NR_CPUS; i++) - seq_printf(m, "%d: %ld\n", i, smtc_cpu_stats[i].timerints); - seq_printf(m, "Self-IPIs by CPU:\n"); - for(i = 0; i < NR_CPUS; i++) - seq_printf(m, "%d: %ld\n", i, smtc_cpu_stats[i].selfipis); - seq_printf(m, "%d Recoveries of \"stolen\" FPU\n", - atomic_read(&smtc_fpu_recoveries)); - return 0; -} - -static int smtc_proc_open(struct inode *inode, struct file *file) -{ - return single_open(file, smtc_proc_show, NULL); -} - -static const struct file_operations smtc_proc_fops = { - .open = smtc_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -void init_smtc_stats(void) -{ - int i; - - for (i=0; i<NR_CPUS; i++) { - smtc_cpu_stats[i].timerints = 0; - smtc_cpu_stats[i].selfipis = 0; - } - - atomic_set(&smtc_fpu_recoveries, 0); - - proc_create("smtc", 0444, NULL, &smtc_proc_fops); -} diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c deleted file mode 100644 index 75a4fd709841..000000000000 --- a/arch/mips/kernel/smtc.c +++ /dev/null @@ -1,1528 +0,0 @@ -/* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * - * Copyright (C) 2004 Mips Technologies, Inc - * Copyright (C) 2008 Kevin D. Kissell - */ - -#include <linux/clockchips.h> -#include <linux/kernel.h> -#include <linux/sched.h> -#include <linux/smp.h> -#include <linux/cpumask.h> -#include <linux/interrupt.h> -#include <linux/kernel_stat.h> -#include <linux/module.h> -#include <linux/ftrace.h> -#include <linux/slab.h> - -#include <asm/cpu.h> -#include <asm/processor.h> -#include <linux/atomic.h> -#include <asm/hardirq.h> -#include <asm/hazards.h> -#include <asm/irq.h> -#include <asm/idle.h> -#include <asm/mmu_context.h> -#include <asm/mipsregs.h> -#include <asm/cacheflush.h> -#include <asm/time.h> -#include <asm/addrspace.h> -#include <asm/smtc.h> -#include <asm/smtc_proc.h> -#include <asm/setup.h> - -/* - * SMTC Kernel needs to manipulate low-level CPU interrupt mask - * in do_IRQ. These are passed in setup_irq_smtc() and stored - * in this table. - */ -unsigned long irq_hwmask[NR_IRQS]; - -#define LOCK_MT_PRA() \ - local_irq_save(flags); \ - mtflags = dmt() - -#define UNLOCK_MT_PRA() \ - emt(mtflags); \ - local_irq_restore(flags) - -#define LOCK_CORE_PRA() \ - local_irq_save(flags); \ - mtflags = dvpe() - -#define UNLOCK_CORE_PRA() \ - evpe(mtflags); \ - local_irq_restore(flags) - -/* - * Data structures purely associated with SMTC parallelism - */ - - -/* - * Table for tracking ASIDs whose lifetime is prolonged. - */ - -asiduse smtc_live_asid[MAX_SMTC_TLBS][MAX_SMTC_ASIDS]; - -/* - * Number of InterProcessor Interrupt (IPI) message buffers to allocate - */ - -#define IPIBUF_PER_CPU 4 - -struct smtc_ipi_q IPIQ[NR_CPUS]; -static struct smtc_ipi_q freeIPIq; - - -/* - * Number of FPU contexts for each VPE - */ - -static int smtc_nconf1[MAX_SMTC_VPES]; - - -/* Forward declarations */ - -void ipi_decode(struct smtc_ipi *); -static void post_direct_ipi(int cpu, struct smtc_ipi *pipi); -static void setup_cross_vpe_interrupts(unsigned int nvpe); -void init_smtc_stats(void); - -/* Global SMTC Status */ - -unsigned int smtc_status; - -/* Boot command line configuration overrides */ - -static int vpe0limit; -static int ipibuffers; -static int nostlb; -static int asidmask; -unsigned long smtc_asid_mask = 0xff; - -static int __init vpe0tcs(char *str) -{ - get_option(&str, &vpe0limit); - - return 1; -} - -static int __init ipibufs(char *str) -{ - get_option(&str, &ipibuffers); - return 1; -} - -static int __init stlb_disable(char *s) -{ - nostlb = 1; - return 1; -} - -static int __init asidmask_set(char *str) -{ - get_option(&str, &asidmask); - switch (asidmask) { - case 0x1: - case 0x3: - case 0x7: - case 0xf: - case 0x1f: - case 0x3f: - case 0x7f: - case 0xff: - smtc_asid_mask = (unsigned long)asidmask; - break; - default: - printk("ILLEGAL ASID mask 0x%x from command line\n", asidmask); - } - return 1; -} - -__setup("vpe0tcs=", vpe0tcs); -__setup("ipibufs=", ipibufs); -__setup("nostlb", stlb_disable); -__setup("asidmask=", asidmask_set); - -#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG - -static int hang_trig; - -static int __init hangtrig_enable(char *s) -{ - hang_trig = 1; - return 1; -} - - -__setup("hangtrig", hangtrig_enable); - -#define DEFAULT_BLOCKED_IPI_LIMIT 32 - -static int timerq_limit = DEFAULT_BLOCKED_IPI_LIMIT; - -static int __init tintq(char *str) -{ - get_option(&str, &timerq_limit); - return 1; -} - -__setup("tintq=", tintq); - -static int imstuckcount[MAX_SMTC_VPES][8]; -/* vpemask represents IM/IE bits of per-VPE Status registers, low-to-high */ -static int vpemask[MAX_SMTC_VPES][8] = { - {0, 0, 1, 0, 0, 0, 0, 1}, - {0, 0, 0, 0, 0, 0, 0, 1} -}; -int tcnoprog[NR_CPUS]; -static atomic_t idle_hook_initialized = ATOMIC_INIT(0); -static int clock_hang_reported[NR_CPUS]; - -#endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */ - -/* - * Configure shared TLB - VPC configuration bit must be set by caller - */ - -static void smtc_configure_tlb(void) -{ - int i, tlbsiz, vpes; - unsigned long mvpconf0; - unsigned long config1val; - - /* Set up ASID preservation table */ - for (vpes=0; vpes<MAX_SMTC_TLBS; vpes++) { - for(i = 0; i < MAX_SMTC_ASIDS; i++) { - smtc_live_asid[vpes][i] = 0; - } - } - mvpconf0 = read_c0_mvpconf0(); - - if ((vpes = ((mvpconf0 & MVPCONF0_PVPE) - >> MVPCONF0_PVPE_SHIFT) + 1) > 1) { - /* If we have multiple VPEs, try to share the TLB */ - if ((mvpconf0 & MVPCONF0_TLBS) && !nostlb) { - /* - * If TLB sizing is programmable, shared TLB - * size is the total available complement. - * Otherwise, we have to take the sum of all - * static VPE TLB entries. - */ - if ((tlbsiz = ((mvpconf0 & MVPCONF0_PTLBE) - >> MVPCONF0_PTLBE_SHIFT)) == 0) { - /* - * If there's more than one VPE, there had better - * be more than one TC, because we need one to bind - * to each VPE in turn to be able to read - * its configuration state! - */ - settc(1); - /* Stop the TC from doing anything foolish */ - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - /* No need to un-Halt - that happens later anyway */ - for (i=0; i < vpes; i++) { - write_tc_c0_tcbind(i); - /* - * To be 100% sure we're really getting the right - * information, we exit the configuration state - * and do an IHB after each rebinding. - */ - write_c0_mvpcontrol( - read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); - mips_ihb(); - /* - * Only count if the MMU Type indicated is TLB - */ - if (((read_vpe_c0_config() & MIPS_CONF_MT) >> 7) == 1) { - config1val = read_vpe_c0_config1(); - tlbsiz += ((config1val >> 25) & 0x3f) + 1; - } - - /* Put core back in configuration state */ - write_c0_mvpcontrol( - read_c0_mvpcontrol() | MVPCONTROL_VPC ); - mips_ihb(); - } - } - write_c0_mvpcontrol(read_c0_mvpcontrol() | MVPCONTROL_STLB); - ehb(); - - /* - * Setup kernel data structures to use software total, - * rather than read the per-VPE Config1 value. The values - * for "CPU 0" gets copied to all the other CPUs as part - * of their initialization in smtc_cpu_setup(). - */ - - /* MIPS32 limits TLB indices to 64 */ - if (tlbsiz > 64) - tlbsiz = 64; - cpu_data[0].tlbsize = current_cpu_data.tlbsize = tlbsiz; - smtc_status |= SMTC_TLB_SHARED; - local_flush_tlb_all(); - - printk("TLB of %d entry pairs shared by %d VPEs\n", - tlbsiz, vpes); - } else { - printk("WARNING: TLB Not Sharable on SMTC Boot!\n"); - } - } -} - - -/* - * Incrementally build the CPU map out of constituent MIPS MT cores, - * using the specified available VPEs and TCs. Plaform code needs - * to ensure that each MIPS MT core invokes this routine on reset, - * one at a time(!). - * - * This version of the build_cpu_map and prepare_cpus routines assumes - * that *all* TCs of a MIPS MT core will be used for Linux, and that - * they will be spread across *all* available VPEs (to minimise the - * loss of efficiency due to exception service serialization). - * An improved version would pick up configuration information and - * possibly leave some TCs/VPEs as "slave" processors. - * - * Use c0_MVPConf0 to find out how many TCs are available, setting up - * cpu_possible_mask and the logical/physical mappings. - */ - -int __init smtc_build_cpu_map(int start_cpu_slot) -{ - int i, ntcs; - - /* - * The CPU map isn't actually used for anything at this point, - * so it's not clear what else we should do apart from set - * everything up so that "logical" = "physical". - */ - ntcs = ((read_c0_mvpconf0() & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; - for (i=start_cpu_slot; i<NR_CPUS && i<ntcs; i++) { - set_cpu_possible(i, true); - __cpu_number_map[i] = i; - __cpu_logical_map[i] = i; - } -#ifdef CONFIG_MIPS_MT_FPAFF - /* Initialize map of CPUs with FPUs */ - cpus_clear(mt_fpu_cpumask); -#endif - - /* One of those TC's is the one booting, and not a secondary... */ - printk("%i available secondary CPU TC(s)\n", i - 1); - - return i; -} - -/* - * Common setup before any secondaries are started - * Make sure all CPUs are in a sensible state before we boot any of the - * secondaries. - * - * For MIPS MT "SMTC" operation, we set up all TCs, spread as evenly - * as possible across the available VPEs. - */ - -static void smtc_tc_setup(int vpe, int tc, int cpu) -{ - static int cp1contexts[MAX_SMTC_VPES]; - - /* - * Make a local copy of the available FPU contexts in order - * to keep track of TCs that can have one. - */ - if (tc == 1) - { - /* - * FIXME: Multi-core SMTC hasn't been tested and the - * maximum number of VPEs may change. - */ - cp1contexts[0] = smtc_nconf1[0] - 1; - cp1contexts[1] = smtc_nconf1[1]; - } - - settc(tc); - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - write_tc_c0_tcstatus((read_tc_c0_tcstatus() - & ~(TCSTATUS_TKSU | TCSTATUS_DA | TCSTATUS_IXMT)) - | TCSTATUS_A); - /* - * TCContext gets an offset from the base of the IPIQ array - * to be used in low-level code to detect the presence of - * an active IPI queue. - */ - write_tc_c0_tccontext((sizeof(struct smtc_ipi_q) * cpu) << 16); - - /* Bind TC to VPE. */ - write_tc_c0_tcbind(vpe); - - /* In general, all TCs should have the same cpu_data indications. */ - memcpy(&cpu_data[cpu], &cpu_data[0], sizeof(struct cpuinfo_mips)); - - /* Check to see if there is a FPU context available for this TC. */ - if (!cp1contexts[vpe]) - cpu_data[cpu].options &= ~MIPS_CPU_FPU; - else - cp1contexts[vpe]--; - - /* Store the TC and VPE into the cpu_data structure. */ - cpu_data[cpu].vpe_id = vpe; - cpu_data[cpu].tc_id = tc; - - /* FIXME: Multi-core SMTC hasn't been tested, but be prepared. */ - cpu_data[cpu].core = (read_vpe_c0_ebase() >> 1) & 0xff; -} - -/* - * Tweak to get Count registers synced as closely as possible. The - * value seems good for 34K-class cores. - */ - -#define CP0_SKEW 8 - -void smtc_prepare_cpus(int cpus) -{ - int i, vpe, tc, ntc, nvpe, tcpervpe[NR_CPUS], slop, cpu; - unsigned long flags; - unsigned long val; - int nipi; - struct smtc_ipi *pipi; - - /* disable interrupts so we can disable MT */ - local_irq_save(flags); - /* disable MT so we can configure */ - dvpe(); - dmt(); - - spin_lock_init(&freeIPIq.lock); - - /* - * We probably don't have as many VPEs as we do SMP "CPUs", - * but it's possible - and in any case we'll never use more! - */ - for (i=0; i<NR_CPUS; i++) { - IPIQ[i].head = IPIQ[i].tail = NULL; - spin_lock_init(&IPIQ[i].lock); - IPIQ[i].depth = 0; - IPIQ[i].resched_flag = 0; /* No reschedules queued initially */ - } - - /* cpu_data index starts at zero */ - cpu = 0; - cpu_data[cpu].vpe_id = 0; - cpu_data[cpu].tc_id = 0; - cpu_data[cpu].core = (read_c0_ebase() >> 1) & 0xff; - cpu++; - - /* Report on boot-time options */ - mips_mt_set_cpuoptions(); - if (vpelimit > 0) - printk("Limit of %d VPEs set\n", vpelimit); - if (tclimit > 0) - printk("Limit of %d TCs set\n", tclimit); - if (nostlb) { - printk("Shared TLB Use Inhibited - UNSAFE for Multi-VPE Operation\n"); - } - if (asidmask) - printk("ASID mask value override to 0x%x\n", asidmask); - - /* Temporary */ -#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG - if (hang_trig) - printk("Logic Analyser Trigger on suspected TC hang\n"); -#endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */ - - /* Put MVPE's into 'configuration state' */ - write_c0_mvpcontrol( read_c0_mvpcontrol() | MVPCONTROL_VPC ); - - val = read_c0_mvpconf0(); - nvpe = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; - if (vpelimit > 0 && nvpe > vpelimit) - nvpe = vpelimit; - ntc = ((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; - if (ntc > NR_CPUS) - ntc = NR_CPUS; - if (tclimit > 0 && ntc > tclimit) - ntc = tclimit; - slop = ntc % nvpe; - for (i = 0; i < nvpe; i++) { - tcpervpe[i] = ntc / nvpe; - if (slop) { - if((slop - i) > 0) tcpervpe[i]++; - } - } - /* Handle command line override for VPE0 */ - if (vpe0limit > ntc) vpe0limit = ntc; - if (vpe0limit > 0) { - int slopslop; - if (vpe0limit < tcpervpe[0]) { - /* Reducing TC count - distribute to others */ - slop = tcpervpe[0] - vpe0limit; - slopslop = slop % (nvpe - 1); - tcpervpe[0] = vpe0limit; - for (i = 1; i < nvpe; i++) { - tcpervpe[i] += slop / (nvpe - 1); - if(slopslop && ((slopslop - (i - 1) > 0))) - tcpervpe[i]++; - } - } else if (vpe0limit > tcpervpe[0]) { - /* Increasing TC count - steal from others */ - slop = vpe0limit - tcpervpe[0]; - slopslop = slop % (nvpe - 1); - tcpervpe[0] = vpe0limit; - for (i = 1; i < nvpe; i++) { - tcpervpe[i] -= slop / (nvpe - 1); - if(slopslop && ((slopslop - (i - 1) > 0))) - tcpervpe[i]--; - } - } - } - - /* Set up shared TLB */ - smtc_configure_tlb(); - - for (tc = 0, vpe = 0 ; (vpe < nvpe) && (tc < ntc) ; vpe++) { - /* Get number of CP1 contexts for each VPE. */ - if (tc == 0) - { - /* - * Do not call settc() for TC0 or the FPU context - * value will be incorrect. Besides, we know that - * we are TC0 anyway. - */ - smtc_nconf1[0] = ((read_vpe_c0_vpeconf1() & - VPECONF1_NCP1) >> VPECONF1_NCP1_SHIFT); - if (nvpe == 2) - { - settc(1); - smtc_nconf1[1] = ((read_vpe_c0_vpeconf1() & - VPECONF1_NCP1) >> VPECONF1_NCP1_SHIFT); - settc(0); - } - } - if (tcpervpe[vpe] == 0) - continue; - if (vpe != 0) - printk(", "); - printk("VPE %d: TC", vpe); - for (i = 0; i < tcpervpe[vpe]; i++) { - /* - * TC 0 is bound to VPE 0 at reset, - * and is presumably executing this - * code. Leave it alone! - */ - if (tc != 0) { - smtc_tc_setup(vpe, tc, cpu); - if (vpe != 0) { - /* - * Set MVP bit (possibly again). Do it - * here to catch CPUs that have no TCs - * bound to the VPE at reset. In that - * case, a TC must be bound to the VPE - * before we can set VPEControl[MVP] - */ - write_vpe_c0_vpeconf0( - read_vpe_c0_vpeconf0() | - VPECONF0_MVP); - } - cpu++; - } - printk(" %d", tc); - tc++; - } - if (vpe != 0) { - /* - * Allow this VPE to control others. - */ - write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | - VPECONF0_MVP); - - /* - * Clear any stale software interrupts from VPE's Cause - */ - write_vpe_c0_cause(0); - - /* - * Clear ERL/EXL of VPEs other than 0 - * and set restricted interrupt enable/mask. - */ - write_vpe_c0_status((read_vpe_c0_status() - & ~(ST0_BEV | ST0_ERL | ST0_EXL | ST0_IM)) - | (STATUSF_IP0 | STATUSF_IP1 | STATUSF_IP7 - | ST0_IE)); - /* - * set config to be the same as vpe0, - * particularly kseg0 coherency alg - */ - write_vpe_c0_config(read_c0_config()); - /* Clear any pending timer interrupt */ - write_vpe_c0_compare(0); - /* Propagate Config7 */ - write_vpe_c0_config7(read_c0_config7()); - write_vpe_c0_count(read_c0_count() + CP0_SKEW); - ehb(); - } - /* enable multi-threading within VPE */ - write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() | VPECONTROL_TE); - /* enable the VPE */ - write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA); - } - - /* - * Pull any physically present but unused TCs out of circulation. - */ - while (tc < (((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1)) { - set_cpu_possible(tc, false); - set_cpu_present(tc, false); - tc++; - } - - /* release config state */ - write_c0_mvpcontrol( read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); - - printk("\n"); - - /* Set up coprocessor affinity CPU mask(s) */ - -#ifdef CONFIG_MIPS_MT_FPAFF - for (tc = 0; tc < ntc; tc++) { - if (cpu_data[tc].options & MIPS_CPU_FPU) - cpu_set(tc, mt_fpu_cpumask); - } -#endif - - /* set up ipi interrupts... */ - - /* If we have multiple VPEs running, set up the cross-VPE interrupt */ - - setup_cross_vpe_interrupts(nvpe); - - /* Set up queue of free IPI "messages". */ - nipi = NR_CPUS * IPIBUF_PER_CPU; - if (ipibuffers > 0) - nipi = ipibuffers; - - pipi = kmalloc(nipi *sizeof(struct smtc_ipi), GFP_KERNEL); - if (pipi == NULL) - panic("kmalloc of IPI message buffers failed"); - else - printk("IPI buffer pool of %d buffers\n", nipi); - for (i = 0; i < nipi; i++) { - smtc_ipi_nq(&freeIPIq, pipi); - pipi++; - } - - /* Arm multithreading and enable other VPEs - but all TCs are Halted */ - emt(EMT_ENABLE); - evpe(EVPE_ENABLE); - local_irq_restore(flags); - /* Initialize SMTC /proc statistics/diagnostics */ - init_smtc_stats(); -} - - -/* - * Setup the PC, SP, and GP of a secondary processor and start it - * running! - * smp_bootstrap is the place to resume from - * __KSTK_TOS(idle) is apparently the stack pointer - * (unsigned long)idle->thread_info the gp - * - */ -void __cpuinit smtc_boot_secondary(int cpu, struct task_struct *idle) -{ - extern u32 kernelsp[NR_CPUS]; - unsigned long flags; - int mtflags; - - LOCK_MT_PRA(); - if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { - dvpe(); - } - settc(cpu_data[cpu].tc_id); - - /* pc */ - write_tc_c0_tcrestart((unsigned long)&smp_bootstrap); - - /* stack pointer */ - kernelsp[cpu] = __KSTK_TOS(idle); - write_tc_gpr_sp(__KSTK_TOS(idle)); - - /* global pointer */ - write_tc_gpr_gp((unsigned long)task_thread_info(idle)); - - smtc_status |= SMTC_MTC_ACTIVE; - write_tc_c0_tchalt(0); - if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { - evpe(EVPE_ENABLE); - } - UNLOCK_MT_PRA(); -} - -void smtc_init_secondary(void) -{ -} - -void smtc_smp_finish(void) -{ - int cpu = smp_processor_id(); - - /* - * Lowest-numbered CPU per VPE starts a clock tick. - * Like per_cpu_trap_init() hack, this assumes that - * SMTC init code assigns TCs consdecutively and - * in ascending order across available VPEs. - */ - if (cpu > 0 && (cpu_data[cpu].vpe_id != cpu_data[cpu - 1].vpe_id)) - write_c0_compare(read_c0_count() + mips_hpt_frequency/HZ); - - local_irq_enable(); - - printk("TC %d going on-line as CPU %d\n", - cpu_data[smp_processor_id()].tc_id, smp_processor_id()); -} - -void smtc_cpus_done(void) -{ -} - -/* - * Support for SMTC-optimized driver IRQ registration - */ - -/* - * SMTC Kernel needs to manipulate low-level CPU interrupt mask - * in do_IRQ. These are passed in setup_irq_smtc() and stored - * in this table. - */ - -int setup_irq_smtc(unsigned int irq, struct irqaction * new, - unsigned long hwmask) -{ -#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG - unsigned int vpe = current_cpu_data.vpe_id; - - vpemask[vpe][irq - MIPS_CPU_IRQ_BASE] = 1; -#endif - irq_hwmask[irq] = hwmask; - - return setup_irq(irq, new); -} - -#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF -/* - * Support for IRQ affinity to TCs - */ - -void smtc_set_irq_affinity(unsigned int irq, cpumask_t affinity) -{ - /* - * If a "fast path" cache of quickly decodable affinity state - * is maintained, this is where it gets done, on a call up - * from the platform affinity code. - */ -} - -void smtc_forward_irq(struct irq_data *d) -{ - unsigned int irq = d->irq; - int target; - - /* - * OK wise guy, now figure out how to get the IRQ - * to be serviced on an authorized "CPU". - * - * Ideally, to handle the situation where an IRQ has multiple - * eligible CPUS, we would maintain state per IRQ that would - * allow a fair distribution of service requests. Since the - * expected use model is any-or-only-one, for simplicity - * and efficiency, we just pick the easiest one to find. - */ - - target = cpumask_first(d->affinity); - - /* - * We depend on the platform code to have correctly processed - * IRQ affinity change requests to ensure that the IRQ affinity - * mask has been purged of bits corresponding to nonexistent and - * offline "CPUs", and to TCs bound to VPEs other than the VPE - * connected to the physical interrupt input for the interrupt - * in question. Otherwise we have a nasty problem with interrupt - * mask management. This is best handled in non-performance-critical - * platform IRQ affinity setting code, to minimize interrupt-time - * checks. - */ - - /* If no one is eligible, service locally */ - if (target >= NR_CPUS) - do_IRQ_no_affinity(irq); - else - smtc_send_ipi(target, IRQ_AFFINITY_IPI, irq); -} - -#endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */ - -/* - * IPI model for SMTC is tricky, because interrupts aren't TC-specific. - * Within a VPE one TC can interrupt another by different approaches. - * The easiest to get right would probably be to make all TCs except - * the target IXMT and set a software interrupt, but an IXMT-based - * scheme requires that a handler must run before a new IPI could - * be sent, which would break the "broadcast" loops in MIPS MT. - * A more gonzo approach within a VPE is to halt the TC, extract - * its Restart, Status, and a couple of GPRs, and program the Restart - * address to emulate an interrupt. - * - * Within a VPE, one can be confident that the target TC isn't in - * a critical EXL state when halted, since the write to the Halt - * register could not have issued on the writing thread if the - * halting thread had EXL set. So k0 and k1 of the target TC - * can be used by the injection code. Across VPEs, one can't - * be certain that the target TC isn't in a critical exception - * state. So we try a two-step process of sending a software - * interrupt to the target VPE, which either handles the event - * itself (if it was the target) or injects the event within - * the VPE. - */ - -static void smtc_ipi_qdump(void) -{ - int i; - struct smtc_ipi *temp; - - for (i = 0; i < NR_CPUS ;i++) { - pr_info("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n", - i, (unsigned)IPIQ[i].head, (unsigned)IPIQ[i].tail, - IPIQ[i].depth); - temp = IPIQ[i].head; - - while (temp != IPIQ[i].tail) { - pr_debug("%d %d %d: ", temp->type, temp->dest, - (int)temp->arg); -#ifdef SMTC_IPI_DEBUG - pr_debug("%u %lu\n", temp->sender, temp->stamp); -#else - pr_debug("\n"); -#endif - temp = temp->flink; - } - } -} - -/* - * The standard atomic.h primitives don't quite do what we want - * here: We need an atomic add-and-return-previous-value (which - * could be done with atomic_add_return and a decrement) and an - * atomic set/zero-and-return-previous-value (which can't really - * be done with the atomic.h primitives). And since this is - * MIPS MT, we can assume that we have LL/SC. - */ -static inline int atomic_postincrement(atomic_t *v) -{ - unsigned long result; - - unsigned long temp; - - __asm__ __volatile__( - "1: ll %0, %2 \n" - " addu %1, %0, 1 \n" - " sc %1, %2 \n" - " beqz %1, 1b \n" - __WEAK_LLSC_MB - : "=&r" (result), "=&r" (temp), "=m" (v->counter) - : "m" (v->counter) - : "memory"); - - return result; -} - -void smtc_send_ipi(int cpu, int type, unsigned int action) -{ - int tcstatus; - struct smtc_ipi *pipi; - unsigned long flags; - int mtflags; - unsigned long tcrestart; - int set_resched_flag = (type == LINUX_SMP_IPI && - action == SMP_RESCHEDULE_YOURSELF); - - if (cpu == smp_processor_id()) { - printk("Cannot Send IPI to self!\n"); - return; - } - if (set_resched_flag && IPIQ[cpu].resched_flag != 0) - return; /* There is a reschedule queued already */ - - /* Set up a descriptor, to be delivered either promptly or queued */ - pipi = smtc_ipi_dq(&freeIPIq); - if (pipi == NULL) { - bust_spinlocks(1); - mips_mt_regdump(dvpe()); - panic("IPI Msg. Buffers Depleted"); - } - pipi->type = type; - pipi->arg = (void *)action; - pipi->dest = cpu; - if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { - /* If not on same VPE, enqueue and send cross-VPE interrupt */ - IPIQ[cpu].resched_flag |= set_resched_flag; - smtc_ipi_nq(&IPIQ[cpu], pipi); - LOCK_CORE_PRA(); - settc(cpu_data[cpu].tc_id); - write_vpe_c0_cause(read_vpe_c0_cause() | C_SW1); - UNLOCK_CORE_PRA(); - } else { - /* - * Not sufficient to do a LOCK_MT_PRA (dmt) here, - * since ASID shootdown on the other VPE may - * collide with this operation. - */ - LOCK_CORE_PRA(); - settc(cpu_data[cpu].tc_id); - /* Halt the targeted TC */ - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - - /* - * Inspect TCStatus - if IXMT is set, we have to queue - * a message. Otherwise, we set up the "interrupt" - * of the other TC - */ - tcstatus = read_tc_c0_tcstatus(); - - if ((tcstatus & TCSTATUS_IXMT) != 0) { - /* - * If we're in the the irq-off version of the wait - * loop, we need to force exit from the wait and - * do a direct post of the IPI. - */ - if (cpu_wait == r4k_wait_irqoff) { - tcrestart = read_tc_c0_tcrestart(); - if (address_is_in_r4k_wait_irqoff(tcrestart)) { - write_tc_c0_tcrestart(__pastwait); - tcstatus &= ~TCSTATUS_IXMT; - write_tc_c0_tcstatus(tcstatus); - goto postdirect; - } - } - /* - * Otherwise we queue the message for the target TC - * to pick up when he does a local_irq_restore() - */ - write_tc_c0_tchalt(0); - UNLOCK_CORE_PRA(); - IPIQ[cpu].resched_flag |= set_resched_flag; - smtc_ipi_nq(&IPIQ[cpu], pipi); - } else { -postdirect: - post_direct_ipi(cpu, pipi); - write_tc_c0_tchalt(0); - UNLOCK_CORE_PRA(); - } - } -} - -/* - * Send IPI message to Halted TC, TargTC/TargVPE already having been set - */ -static void post_direct_ipi(int cpu, struct smtc_ipi *pipi) -{ - struct pt_regs *kstack; - unsigned long tcstatus; - unsigned long tcrestart; - extern u32 kernelsp[NR_CPUS]; - extern void __smtc_ipi_vector(void); -//printk("%s: on %d for %d\n", __func__, smp_processor_id(), cpu); - - /* Extract Status, EPC from halted TC */ - tcstatus = read_tc_c0_tcstatus(); - tcrestart = read_tc_c0_tcrestart(); - /* If TCRestart indicates a WAIT instruction, advance the PC */ - if ((tcrestart & 0x80000000) - && ((*(unsigned int *)tcrestart & 0xfe00003f) == 0x42000020)) { - tcrestart += 4; - } - /* - * Save on TC's future kernel stack - * - * CU bit of Status is indicator that TC was - * already running on a kernel stack... - */ - if (tcstatus & ST0_CU0) { - /* Note that this "- 1" is pointer arithmetic */ - kstack = ((struct pt_regs *)read_tc_gpr_sp()) - 1; - } else { - kstack = ((struct pt_regs *)kernelsp[cpu]) - 1; - } - - kstack->cp0_epc = (long)tcrestart; - /* Save TCStatus */ - kstack->cp0_tcstatus = tcstatus; - /* Pass token of operation to be performed kernel stack pad area */ - kstack->pad0[4] = (unsigned long)pipi; - /* Pass address of function to be called likewise */ - kstack->pad0[5] = (unsigned long)&ipi_decode; - /* Set interrupt exempt and kernel mode */ - tcstatus |= TCSTATUS_IXMT; - tcstatus &= ~TCSTATUS_TKSU; - write_tc_c0_tcstatus(tcstatus); - ehb(); - /* Set TC Restart address to be SMTC IPI vector */ - write_tc_c0_tcrestart(__smtc_ipi_vector); -} - -static void ipi_resched_interrupt(void) -{ - scheduler_ipi(); -} - -static void ipi_call_interrupt(void) -{ - /* Invoke generic function invocation code in smp.c */ - smp_call_function_interrupt(); -} - -DECLARE_PER_CPU(struct clock_event_device, mips_clockevent_device); - -static void __irq_entry smtc_clock_tick_interrupt(void) -{ - unsigned int cpu = smp_processor_id(); - struct clock_event_device *cd; - int irq = MIPS_CPU_IRQ_BASE + 1; - - irq_enter(); - kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq)); - cd = &per_cpu(mips_clockevent_device, cpu); - cd->event_handler(cd); - irq_exit(); -} - -void ipi_decode(struct smtc_ipi *pipi) -{ - void *arg_copy = pipi->arg; - int type_copy = pipi->type; - - smtc_ipi_nq(&freeIPIq, pipi); - - switch (type_copy) { - case SMTC_CLOCK_TICK: - smtc_clock_tick_interrupt(); - break; - - case LINUX_SMP_IPI: - switch ((int)arg_copy) { - case SMP_RESCHEDULE_YOURSELF: - ipi_resched_interrupt(); - break; - case SMP_CALL_FUNCTION: - ipi_call_interrupt(); - break; - default: - printk("Impossible SMTC IPI Argument %p\n", arg_copy); - break; - } - break; -#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF - case IRQ_AFFINITY_IPI: - /* - * Accept a "forwarded" interrupt that was initially - * taken by a TC who doesn't have affinity for the IRQ. - */ - do_IRQ_no_affinity((int)arg_copy); - break; -#endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */ - default: - printk("Impossible SMTC IPI Type 0x%x\n", type_copy); - break; - } -} - -/* - * Similar to smtc_ipi_replay(), but invoked from context restore, - * so it reuses the current exception frame rather than set up a - * new one with self_ipi. - */ - -void deferred_smtc_ipi(void) -{ - int cpu = smp_processor_id(); - - /* - * Test is not atomic, but much faster than a dequeue, - * and the vast majority of invocations will have a null queue. - * If irq_disabled when this was called, then any IPIs queued - * after we test last will be taken on the next irq_enable/restore. - * If interrupts were enabled, then any IPIs added after the - * last test will be taken directly. - */ - - while (IPIQ[cpu].head != NULL) { - struct smtc_ipi_q *q = &IPIQ[cpu]; - struct smtc_ipi *pipi; - unsigned long flags; - - /* - * It may be possible we'll come in with interrupts - * already enabled. - */ - local_irq_save(flags); - spin_lock(&q->lock); - pipi = __smtc_ipi_dq(q); - spin_unlock(&q->lock); - if (pipi != NULL) { - if (pipi->type == LINUX_SMP_IPI && - (int)pipi->arg == SMP_RESCHEDULE_YOURSELF) - IPIQ[cpu].resched_flag = 0; - ipi_decode(pipi); - } - /* - * The use of the __raw_local restore isn't - * as obviously necessary here as in smtc_ipi_replay(), - * but it's more efficient, given that we're already - * running down the IPI queue. - */ - __arch_local_irq_restore(flags); - } -} - -/* - * Cross-VPE interrupts in the SMTC prototype use "software interrupts" - * set via cross-VPE MTTR manipulation of the Cause register. It would be - * in some regards preferable to have external logic for "doorbell" hardware - * interrupts. - */ - -static int cpu_ipi_irq = MIPS_CPU_IRQ_BASE + MIPS_CPU_IPI_IRQ; - -static irqreturn_t ipi_interrupt(int irq, void *dev_idm) -{ - int my_vpe = cpu_data[smp_processor_id()].vpe_id; - int my_tc = cpu_data[smp_processor_id()].tc_id; - int cpu; - struct smtc_ipi *pipi; - unsigned long tcstatus; - int sent; - unsigned long flags; - unsigned int mtflags; - unsigned int vpflags; - - /* - * So long as cross-VPE interrupts are done via - * MFTR/MTTR read-modify-writes of Cause, we need - * to stop other VPEs whenever the local VPE does - * anything similar. - */ - local_irq_save(flags); - vpflags = dvpe(); - clear_c0_cause(0x100 << MIPS_CPU_IPI_IRQ); - set_c0_status(0x100 << MIPS_CPU_IPI_IRQ); - irq_enable_hazard(); - evpe(vpflags); - local_irq_restore(flags); - - /* - * Cross-VPE Interrupt handler: Try to directly deliver IPIs - * queued for TCs on this VPE other than the current one. - * Return-from-interrupt should cause us to drain the queue - * for the current TC, so we ought not to have to do it explicitly here. - */ - - for_each_online_cpu(cpu) { - if (cpu_data[cpu].vpe_id != my_vpe) - continue; - - pipi = smtc_ipi_dq(&IPIQ[cpu]); - if (pipi != NULL) { - if (cpu_data[cpu].tc_id != my_tc) { - sent = 0; - LOCK_MT_PRA(); - settc(cpu_data[cpu].tc_id); - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - tcstatus = read_tc_c0_tcstatus(); - if ((tcstatus & TCSTATUS_IXMT) == 0) { - post_direct_ipi(cpu, pipi); - sent = 1; - } - write_tc_c0_tchalt(0); - UNLOCK_MT_PRA(); - if (!sent) { - smtc_ipi_req(&IPIQ[cpu], pipi); - } - } else { - /* - * ipi_decode() should be called - * with interrupts off - */ - local_irq_save(flags); - if (pipi->type == LINUX_SMP_IPI && - (int)pipi->arg == SMP_RESCHEDULE_YOURSELF) - IPIQ[cpu].resched_flag = 0; - ipi_decode(pipi); - local_irq_restore(flags); - } - } - } - - return IRQ_HANDLED; -} - -static void ipi_irq_dispatch(void) -{ - do_IRQ(cpu_ipi_irq); -} - -static struct irqaction irq_ipi = { - .handler = ipi_interrupt, - .flags = IRQF_PERCPU, - .name = "SMTC_IPI" -}; - -static void setup_cross_vpe_interrupts(unsigned int nvpe) -{ - if (nvpe < 1) - return; - - if (!cpu_has_vint) - panic("SMTC Kernel requires Vectored Interrupt support"); - - set_vi_handler(MIPS_CPU_IPI_IRQ, ipi_irq_dispatch); - - setup_irq_smtc(cpu_ipi_irq, &irq_ipi, (0x100 << MIPS_CPU_IPI_IRQ)); - - irq_set_handler(cpu_ipi_irq, handle_percpu_irq); -} - -/* - * SMTC-specific hacks invoked from elsewhere in the kernel. - */ - - /* - * smtc_ipi_replay is called from raw_local_irq_restore - */ - -void smtc_ipi_replay(void) -{ - unsigned int cpu = smp_processor_id(); - - /* - * To the extent that we've ever turned interrupts off, - * we may have accumulated deferred IPIs. This is subtle. - * we should be OK: If we pick up something and dispatch - * it here, that's great. If we see nothing, but concurrent - * with this operation, another TC sends us an IPI, IXMT - * is clear, and we'll handle it as a real pseudo-interrupt - * and not a pseudo-pseudo interrupt. The important thing - * is to do the last check for queued message *after* the - * re-enabling of interrupts. - */ - while (IPIQ[cpu].head != NULL) { - struct smtc_ipi_q *q = &IPIQ[cpu]; - struct smtc_ipi *pipi; - unsigned long flags; - - /* - * It's just possible we'll come in with interrupts - * already enabled. - */ - local_irq_save(flags); - - spin_lock(&q->lock); - pipi = __smtc_ipi_dq(q); - spin_unlock(&q->lock); - /* - ** But use a raw restore here to avoid recursion. - */ - __arch_local_irq_restore(flags); - - if (pipi) { - self_ipi(pipi); - smtc_cpu_stats[cpu].selfipis++; - } - } -} - -EXPORT_SYMBOL(smtc_ipi_replay); - -void smtc_idle_loop_hook(void) -{ -#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG - int im; - int flags; - int mtflags; - int bit; - int vpe; - int tc; - int hook_ntcs; - /* - * printk within DMT-protected regions can deadlock, - * so buffer diagnostic messages for later output. - */ - char *pdb_msg; - char id_ho_db_msg[768]; /* worst-case use should be less than 700 */ - - if (atomic_read(&idle_hook_initialized) == 0) { /* fast test */ - if (atomic_add_return(1, &idle_hook_initialized) == 1) { - int mvpconf0; - /* Tedious stuff to just do once */ - mvpconf0 = read_c0_mvpconf0(); - hook_ntcs = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; - if (hook_ntcs > NR_CPUS) - hook_ntcs = NR_CPUS; - for (tc = 0; tc < hook_ntcs; tc++) { - tcnoprog[tc] = 0; - clock_hang_reported[tc] = 0; - } - for (vpe = 0; vpe < 2; vpe++) - for (im = 0; im < 8; im++) - imstuckcount[vpe][im] = 0; - printk("Idle loop test hook initialized for %d TCs\n", hook_ntcs); - atomic_set(&idle_hook_initialized, 1000); - } else { - /* Someone else is initializing in parallel - let 'em finish */ - while (atomic_read(&idle_hook_initialized) < 1000) - ; - } - } - - /* Have we stupidly left IXMT set somewhere? */ - if (read_c0_tcstatus() & 0x400) { - write_c0_tcstatus(read_c0_tcstatus() & ~0x400); - ehb(); - printk("Dangling IXMT in cpu_idle()\n"); - } - - /* Have we stupidly left an IM bit turned off? */ -#define IM_LIMIT 2000 - local_irq_save(flags); - mtflags = dmt(); - pdb_msg = &id_ho_db_msg[0]; - im = read_c0_status(); - vpe = current_cpu_data.vpe_id; - for (bit = 0; bit < 8; bit++) { - /* - * In current prototype, I/O interrupts - * are masked for VPE > 0 - */ - if (vpemask[vpe][bit]) { - if (!(im & (0x100 << bit))) - imstuckcount[vpe][bit]++; - else - imstuckcount[vpe][bit] = 0; - if (imstuckcount[vpe][bit] > IM_LIMIT) { - set_c0_status(0x100 << bit); - ehb(); - imstuckcount[vpe][bit] = 0; - pdb_msg += sprintf(pdb_msg, - "Dangling IM %d fixed for VPE %d\n", bit, - vpe); - } - } - } - - emt(mtflags); - local_irq_restore(flags); - if (pdb_msg != &id_ho_db_msg[0]) - printk("CPU%d: %s", smp_processor_id(), id_ho_db_msg); -#endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */ - - smtc_ipi_replay(); -} - -void smtc_soft_dump(void) -{ - int i; - - printk("Counter Interrupts taken per CPU (TC)\n"); - for (i=0; i < NR_CPUS; i++) { - printk("%d: %ld\n", i, smtc_cpu_stats[i].timerints); - } - printk("Self-IPI invocations:\n"); - for (i=0; i < NR_CPUS; i++) { - printk("%d: %ld\n", i, smtc_cpu_stats[i].selfipis); - } - smtc_ipi_qdump(); - printk("%d Recoveries of \"stolen\" FPU\n", - atomic_read(&smtc_fpu_recoveries)); -} - - -/* - * TLB management routines special to SMTC - */ - -void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) -{ - unsigned long flags, mtflags, tcstat, prevhalt, asid; - int tlb, i; - - /* - * It would be nice to be able to use a spinlock here, - * but this is invoked from within TLB flush routines - * that protect themselves with DVPE, so if a lock is - * held by another TC, it'll never be freed. - * - * DVPE/DMT must not be done with interrupts enabled, - * so even so most callers will already have disabled - * them, let's be really careful... - */ - - local_irq_save(flags); - if (smtc_status & SMTC_TLB_SHARED) { - mtflags = dvpe(); - tlb = 0; - } else { - mtflags = dmt(); - tlb = cpu_data[cpu].vpe_id; - } - asid = asid_cache(cpu); - - do { - if (!((asid += ASID_INC) & ASID_MASK) ) { - if (cpu_has_vtag_icache) - flush_icache_all(); - /* Traverse all online CPUs (hack requires contiguous range) */ - for_each_online_cpu(i) { - /* - * We don't need to worry about our own CPU, nor those of - * CPUs who don't share our TLB. - */ - if ((i != smp_processor_id()) && - ((smtc_status & SMTC_TLB_SHARED) || - (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))) { - settc(cpu_data[i].tc_id); - prevhalt = read_tc_c0_tchalt() & TCHALT_H; - if (!prevhalt) { - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - } - tcstat = read_tc_c0_tcstatus(); - smtc_live_asid[tlb][(tcstat & ASID_MASK)] |= (asiduse)(0x1 << i); - if (!prevhalt) - write_tc_c0_tchalt(0); - } - } - if (!asid) /* fix version if needed */ - asid = ASID_FIRST_VERSION; - local_flush_tlb_all(); /* start new asid cycle */ - } - } while (smtc_live_asid[tlb][(asid & ASID_MASK)]); - - /* - * SMTC shares the TLB within VPEs and possibly across all VPEs. - */ - for_each_online_cpu(i) { - if ((smtc_status & SMTC_TLB_SHARED) || - (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id)) - cpu_context(i, mm) = asid_cache(i) = asid; - } - - if (smtc_status & SMTC_TLB_SHARED) - evpe(mtflags); - else - emt(mtflags); - local_irq_restore(flags); -} - -/* - * Invoked from macros defined in mmu_context.h - * which must already have disabled interrupts - * and done a DVPE or DMT as appropriate. - */ - -void smtc_flush_tlb_asid(unsigned long asid) -{ - int entry; - unsigned long ehi; - - entry = read_c0_wired(); - - /* Traverse all non-wired entries */ - while (entry < current_cpu_data.tlbsize) { - write_c0_index(entry); - ehb(); - tlb_read(); - ehb(); - ehi = read_c0_entryhi(); - if ((ehi & ASID_MASK) == asid) { - /* - * Invalidate only entries with specified ASID, - * makiing sure all entries differ. - */ - write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1))); - write_c0_entrylo0(0); - write_c0_entrylo1(0); - mtc0_tlbw_hazard(); - tlb_write_indexed(); - } - entry++; - } - write_c0_index(PARKED_INDEX); - tlbw_use_hazard(); -} - -/* - * Support for single-threading cache flush operations. - */ - -static int halt_state_save[NR_CPUS]; - -/* - * To really, really be sure that nothing is being done - * by other TCs, halt them all. This code assumes that - * a DVPE has already been done, so while their Halted - * state is theoretically architecturally unstable, in - * practice, it's not going to change while we're looking - * at it. - */ - -void smtc_cflush_lockdown(void) -{ - int cpu; - - for_each_online_cpu(cpu) { - if (cpu != smp_processor_id()) { - settc(cpu_data[cpu].tc_id); - halt_state_save[cpu] = read_tc_c0_tchalt(); - write_tc_c0_tchalt(TCHALT_H); - } - } - mips_ihb(); -} - -/* It would be cheating to change the cpu_online states during a flush! */ - -void smtc_cflush_release(void) -{ - int cpu; - - /* - * Start with a hazard barrier to ensure - * that all CACHE ops have played through. - */ - mips_ihb(); - - for_each_online_cpu(cpu) { - if (cpu != smp_processor_id()) { - settc(cpu_data[cpu].tc_id); - write_tc_c0_tchalt(halt_state_save[cpu]); - } - } - mips_ihb(); -} diff --git a/arch/mips/kernel/spinlock_test.c b/arch/mips/kernel/spinlock_test.c index 39f7ab7b0426..90f53e041a38 100644 --- a/arch/mips/kernel/spinlock_test.c +++ b/arch/mips/kernel/spinlock_test.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/init.h> #include <linux/kthread.h> #include <linux/hrtimer.h> @@ -5,7 +6,7 @@ #include <linux/debugfs.h> #include <linux/export.h> #include <linux/spinlock.h> - +#include <asm/debug.h> static int ss_get(void *data, u64 *val) { @@ -34,7 +35,7 @@ static int ss_get(void *data, u64 *val) return 0; } -DEFINE_SIMPLE_ATTRIBUTE(fops_ss, ss_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_ss, ss_get, NULL, "%llu\n"); @@ -113,29 +114,14 @@ static int multi_get(void *data, u64 *val) return 0; } -DEFINE_SIMPLE_ATTRIBUTE(fops_multi, multi_get, NULL, "%llu\n"); - +DEFINE_DEBUGFS_ATTRIBUTE(fops_multi, multi_get, NULL, "%llu\n"); -extern struct dentry *mips_debugfs_dir; static int __init spinlock_test(void) { - struct dentry *d; - - if (!mips_debugfs_dir) - return -ENODEV; - - d = debugfs_create_file("spin_single", S_IRUGO, - mips_debugfs_dir, NULL, - &fops_ss); - if (!d) - return -ENOMEM; - - d = debugfs_create_file("spin_multi", S_IRUGO, - mips_debugfs_dir, NULL, - &fops_multi); - if (!d) - return -ENOMEM; - + debugfs_create_file_unsafe("spin_single", S_IRUGO, mips_debugfs_dir, NULL, + &fops_ss); + debugfs_create_file_unsafe("spin_multi", S_IRUGO, mips_debugfs_dir, NULL, + &fops_multi); return 0; } device_initcall(spinlock_test); diff --git a/arch/mips/kernel/spram.c b/arch/mips/kernel/spram.c index 6af08d896e20..dd31e3fffd24 100644 --- a/arch/mips/kernel/spram.c +++ b/arch/mips/kernel/spram.c @@ -1,14 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * MIPS SPRAM support * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * * Copyright (C) 2007, 2008 MIPS Technologies, Inc. */ -#include <linux/init.h> #include <linux/kernel.h> #include <linux/ptrace.h> #include <linux/stddef.h> @@ -17,6 +12,7 @@ #include <asm/mipsregs.h> #include <asm/r4kcache.h> #include <asm/hazards.h> +#include <asm/spram.h> /* * These definitions are correct for the 24K/34K/74K SPRAM sample @@ -30,14 +26,10 @@ #define ERRCTL_SPRAM (1 << 28) -/* errctl access */ -#define read_c0_errctl(x) read_c0_ecc(x) -#define write_c0_errctl(x) write_c0_ecc(x) - /* * Different semantics to the set_c0_* function built by __BUILD_SET_C0 */ -static __cpuinit unsigned int bis_c0_errctl(unsigned int set) +static unsigned int bis_c0_errctl(unsigned int set) { unsigned int res; res = read_c0_errctl(); @@ -45,7 +37,7 @@ static __cpuinit unsigned int bis_c0_errctl(unsigned int set) return res; } -static __cpuinit void ispram_store_tag(unsigned int offset, unsigned int data) +static void ispram_store_tag(unsigned int offset, unsigned int data) { unsigned int errctl; @@ -64,7 +56,7 @@ static __cpuinit void ispram_store_tag(unsigned int offset, unsigned int data) } -static __cpuinit unsigned int ispram_load_tag(unsigned int offset) +static unsigned int ispram_load_tag(unsigned int offset) { unsigned int data; unsigned int errctl; @@ -82,7 +74,7 @@ static __cpuinit unsigned int ispram_load_tag(unsigned int offset) return data; } -static __cpuinit void dspram_store_tag(unsigned int offset, unsigned int data) +static void dspram_store_tag(unsigned int offset, unsigned int data) { unsigned int errctl; @@ -98,7 +90,7 @@ static __cpuinit void dspram_store_tag(unsigned int offset, unsigned int data) } -static __cpuinit unsigned int dspram_load_tag(unsigned int offset) +static unsigned int dspram_load_tag(unsigned int offset) { unsigned int data; unsigned int errctl; @@ -115,7 +107,7 @@ static __cpuinit unsigned int dspram_load_tag(unsigned int offset) return data; } -static __cpuinit void probe_spram(char *type, +static void probe_spram(char *type, unsigned int base, unsigned int (*read)(unsigned int), void (*write)(unsigned int, unsigned int)) @@ -196,23 +188,29 @@ static __cpuinit void probe_spram(char *type, offset += 2 * SPRAM_TAG_STRIDE; } } -void __cpuinit spram_config(void) +void spram_config(void) { - struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int config0; - switch (c->cputype) { + switch (current_cpu_type()) { case CPU_24K: case CPU_34K: case CPU_74K: case CPU_1004K: + case CPU_1074K: + case CPU_INTERAPTIV: + case CPU_PROAPTIV: + case CPU_P5600: + case CPU_QEMU_GENERIC: + case CPU_I6400: + case CPU_P6600: config0 = read_c0_config(); /* FIXME: addresses are Malta specific */ - if (config0 & (1<<24)) { + if (config0 & MIPS_CONF_ISP) { probe_spram("ISPRAM", 0x1c000000, &ispram_load_tag, &ispram_store_tag); } - if (config0 & (1<<23)) + if (config0 & MIPS_CONF_DSP) probe_spram("DSPRAM", 0x1c100000, &dspram_load_tag, &dspram_store_tag); } diff --git a/arch/mips/kernel/stacktrace.c b/arch/mips/kernel/stacktrace.c index 1ba775d24d38..f2e72094051b 100644 --- a/arch/mips/kernel/stacktrace.c +++ b/arch/mips/kernel/stacktrace.c @@ -1,9 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Stack trace management functions * * Copyright (C) 2006 Atsushi Nemoto <anemo@mba.ocn.ne.jp> */ #include <linux/sched.h> +#include <linux/sched/debug.h> +#include <linux/sched/task_stack.h> #include <linux/stacktrace.h> #include <linux/export.h> #include <asm/stacktrace.h> @@ -12,14 +15,15 @@ * Save stack-backtrace addresses into a stack_trace buffer: */ static void save_raw_context_stack(struct stack_trace *trace, - unsigned long reg29) + unsigned long reg29, int savesched) { unsigned long *sp = (unsigned long *)reg29; unsigned long addr; while (!kstack_end(sp)) { addr = *sp++; - if (__kernel_text_address(addr)) { + if (__kernel_text_address(addr) && + (savesched || !in_sched_functions(addr))) { if (trace->skip > 0) trace->skip--; else @@ -31,7 +35,7 @@ static void save_raw_context_stack(struct stack_trace *trace, } static void save_context_stack(struct stack_trace *trace, - struct task_struct *tsk, struct pt_regs *regs) + struct task_struct *tsk, struct pt_regs *regs, int savesched) { unsigned long sp = regs->regs[29]; #ifdef CONFIG_KALLSYMS @@ -43,20 +47,22 @@ static void save_context_stack(struct stack_trace *trace, (unsigned long)task_stack_page(tsk); if (stack_page && sp >= stack_page && sp <= stack_page + THREAD_SIZE - 32) - save_raw_context_stack(trace, sp); + save_raw_context_stack(trace, sp, savesched); return; } do { - if (trace->skip > 0) - trace->skip--; - else - trace->entries[trace->nr_entries++] = pc; - if (trace->nr_entries >= trace->max_entries) - break; + if (savesched || !in_sched_functions(pc)) { + if (trace->skip > 0) + trace->skip--; + else + trace->entries[trace->nr_entries++] = pc; + if (trace->nr_entries >= trace->max_entries) + break; + } pc = unwind_stack(tsk, &sp, pc, &ra); } while (pc); #else - save_raw_context_stack(trace, sp); + save_raw_context_stack(trace, sp, savesched); #endif } @@ -82,6 +88,6 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) regs->cp0_epc = tsk->thread.reg31; } else prepare_frametrace(regs); - save_context_stack(trace, tsk, regs); + save_context_stack(trace, tsk, regs, tsk == current); } EXPORT_SYMBOL_GPL(save_stack_trace_tsk); diff --git a/arch/mips/kernel/sync-r4k.c b/arch/mips/kernel/sync-r4k.c index 1ff43d5ac2c4..39156592582e 100644 --- a/arch/mips/kernel/sync-r4k.c +++ b/arch/mips/kernel/sync-r4k.c @@ -1,151 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Count register synchronisation. * - * All CPUs will have their count registers synchronised to the CPU0 next time - * value. This can cause a small timewarp for CPU0. All other CPU's should - * not have done anything significant (but they may have had interrupts - * enabled briefly - prom_smp_finish() should not be responsible for enabling - * interrupts...) - * - * FIXME: broken for SMTC + * Derived from arch/x86/kernel/tsc_sync.c + * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar */ #include <linux/kernel.h> -#include <linux/init.h> #include <linux/irqflags.h> #include <linux/cpumask.h> +#include <linux/atomic.h> +#include <linux/nmi.h> +#include <linux/smp.h> +#include <linux/spinlock.h> #include <asm/r4k-timer.h> -#include <linux/atomic.h> -#include <asm/barrier.h> #include <asm/mipsregs.h> +#include <asm/time.h> + +#define COUNTON 100 +#define NR_LOOPS 3 +#define LOOP_TIMEOUT 20 + +/* + * Entry/exit counters that make sure that both CPUs + * run the measurement code at once: + */ +static atomic_t start_count; +static atomic_t stop_count; +static atomic_t test_runs; + +/* + * We use a raw spinlock in this exceptional case, because + * we want to have the fastest, inlined, non-debug version + * of a critical section, to be able to prove counter time-warps: + */ +static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; + +static uint32_t last_counter; +static uint32_t max_warp; +static int nr_warps; +static int random_warps; + +/* + * Counter warp measurement loop running on both CPUs. + */ +static uint32_t check_counter_warp(void) +{ + uint32_t start, now, prev, end, cur_max_warp = 0; + int i, cur_warps = 0; + + start = read_c0_count(); + end = start + (uint32_t) mips_hpt_frequency / 1000 * LOOP_TIMEOUT; -static atomic_t __cpuinitdata count_start_flag = ATOMIC_INIT(0); -static atomic_t __cpuinitdata count_count_start = ATOMIC_INIT(0); -static atomic_t __cpuinitdata count_count_stop = ATOMIC_INIT(0); -static atomic_t __cpuinitdata count_reference = ATOMIC_INIT(0); + for (i = 0; ; i++) { + /* + * We take the global lock, measure counter, save the + * previous counter that was measured (possibly on + * another CPU) and update the previous counter timestamp. + */ + arch_spin_lock(&sync_lock); + prev = last_counter; + now = read_c0_count(); + last_counter = now; + arch_spin_unlock(&sync_lock); -#define COUNTON 100 -#define NR_LOOPS 5 + /* + * Be nice every now and then (and also check whether + * measurement is done [we also insert a 10 million + * loops safety exit, so we dont lock up in case the + * counter is totally broken]): + */ + if (unlikely(!(i & 7))) { + if (now > end || i > 10000000) + break; + cpu_relax(); + touch_nmi_watchdog(); + } + /* + * Outside the critical section we can now see whether + * we saw a time-warp of the counter going backwards: + */ + if (unlikely(prev > now)) { + arch_spin_lock(&sync_lock); + max_warp = max(max_warp, prev - now); + cur_max_warp = max_warp; + /* + * Check whether this bounces back and forth. Only + * one CPU should observe time going backwards. + */ + if (cur_warps != nr_warps) + random_warps++; + nr_warps++; + cur_warps = nr_warps; + arch_spin_unlock(&sync_lock); + } + } + WARN(!(now-start), + "Warning: zero counter calibration delta: %d [max: %d]\n", + now-start, end-start); + return cur_max_warp; +} -void __cpuinit synchronise_count_master(int cpu) +/* + * The freshly booted CPU initiates this via an async SMP function call. + */ +static void check_counter_sync_source(void *__cpu) { - int i; - unsigned long flags; - unsigned int initcount; + unsigned int cpu = (unsigned long)__cpu; + int cpus = 2; + + atomic_set(&test_runs, NR_LOOPS); +retry: + /* Wait for the target to start. */ + while (atomic_read(&start_count) != cpus - 1) + cpu_relax(); -#ifdef CONFIG_MIPS_MT_SMTC /* - * SMTC needs to synchronise per VPE, not per CPU - * ignore for now + * Trigger the target to continue into the measurement too: */ - return; -#endif + atomic_inc(&start_count); + + check_counter_warp(); - printk(KERN_INFO "Synchronize counters for CPU %u: ", cpu); + while (atomic_read(&stop_count) != cpus-1) + cpu_relax(); - local_irq_save(flags); + /* + * If the test was successful set the number of runs to zero and + * stop. If not, decrement the number of runs an check if we can + * retry. In case of random warps no retry is attempted. + */ + if (!nr_warps) { + atomic_set(&test_runs, 0); + + pr_info("Counter synchronization [CPU#%d -> CPU#%u]: passed\n", + smp_processor_id(), cpu); + } else if (atomic_dec_and_test(&test_runs) || random_warps) { + /* Force it to 0 if random warps brought us here */ + atomic_set(&test_runs, 0); + + pr_info("Counter synchronization [CPU#%d -> CPU#%u]:\n", + smp_processor_id(), cpu); + pr_info("Measured %d cycles counter warp between CPUs", max_warp); + if (random_warps) + pr_warn("Counter warped randomly between CPUs\n"); + } /* - * Notify the slaves that it's time to start + * Reset it - just in case we boot another CPU later: */ - atomic_set(&count_reference, read_c0_count()); - atomic_set(&count_start_flag, cpu); - smp_wmb(); + atomic_set(&start_count, 0); + random_warps = 0; + nr_warps = 0; + max_warp = 0; + last_counter = 0; - /* Count will be initialised to current timer for all CPU's */ - initcount = read_c0_count(); + /* + * Let the target continue with the bootup: + */ + atomic_inc(&stop_count); /* - * We loop a few times to get a primed instruction cache, - * then the last pass is more or less synchronised and - * the master and slaves each set their cycle counters to a known - * value all at once. This reduces the chance of having random offsets - * between the processors, and guarantees that the maximum - * delay between the cycle counters is never bigger than - * the latency of information-passing (cachelines) between - * two CPUs. + * Retry, if there is a chance to do so. */ + if (atomic_read(&test_runs) > 0) + goto retry; +} - for (i = 0; i < NR_LOOPS; i++) { - /* slaves loop on '!= 2' */ - while (atomic_read(&count_count_start) != 1) - mb(); - atomic_set(&count_count_stop, 0); - smp_wmb(); +/* + * Freshly booted CPUs call into this: + */ +void synchronise_count_slave(int cpu) +{ + uint32_t cur_max_warp, gbl_max_warp, count; + int cpus = 2; - /* this lets the slaves write their count register */ - atomic_inc(&count_count_start); + if (!cpu_has_counter || !mips_hpt_frequency) + return; - /* - * Everyone initialises count in the last loop: - */ - if (i == NR_LOOPS-1) - write_c0_count(initcount); + /* Kick the control CPU into the counter synchronization function */ + smp_call_function_single(cpumask_first(cpu_online_mask), + check_counter_sync_source, + (unsigned long *)(unsigned long)cpu, 0); +retry: + /* + * Register this CPU's participation and wait for the + * source CPU to start the measurement: + */ + atomic_inc(&start_count); + while (atomic_read(&start_count) != cpus) + cpu_relax(); - /* - * Wait for all slaves to leave the synchronization point: - */ - while (atomic_read(&count_count_stop) != 1) - mb(); - atomic_set(&count_count_start, 0); - smp_wmb(); - atomic_inc(&count_count_stop); - } - /* Arrange for an interrupt in a short while */ - write_c0_compare(read_c0_count() + COUNTON); - atomic_set(&count_start_flag, 0); + cur_max_warp = check_counter_warp(); - local_irq_restore(flags); + /* + * Store the maximum observed warp value for a potential retry: + */ + gbl_max_warp = max_warp; /* - * i386 code reported the skew here, but the - * count registers were almost certainly out of sync - * so no point in alarming people + * Ok, we are done: */ - printk("done.\n"); -} + atomic_inc(&stop_count); -void __cpuinit synchronise_count_slave(int cpu) -{ - int i; - unsigned int initcount; + /* + * Wait for the source CPU to print stuff: + */ + while (atomic_read(&stop_count) != cpus) + cpu_relax(); -#ifdef CONFIG_MIPS_MT_SMTC /* - * SMTC needs to synchronise per VPE, not per CPU - * ignore for now + * Reset it for the next sync test: */ - return; -#endif + atomic_set(&stop_count, 0); /* - * Not every cpu is online at the time this gets called, - * so we first wait for the master to say everyone is ready + * Check the number of remaining test runs. If not zero, the test + * failed and a retry with adjusted counter is possible. If zero the + * test was either successful or failed terminally. */ + if (!atomic_read(&test_runs)) { + /* Arrange for an interrupt in a short while */ + write_c0_compare(read_c0_count() + COUNTON); + return; + } - while (atomic_read(&count_start_flag) != cpu) - mb(); + /* + * If the warp value of this CPU is 0, then the other CPU + * observed time going backwards so this counter was ahead and + * needs to move backwards. + */ + if (!cur_max_warp) + cur_max_warp = -gbl_max_warp; - /* Count will be initialised to next expire for all CPU's */ - initcount = atomic_read(&count_reference); + count = read_c0_count(); + count += cur_max_warp; + write_c0_count(count); - for (i = 0; i < NR_LOOPS; i++) { - atomic_inc(&count_count_start); - while (atomic_read(&count_count_start) != 2) - mb(); + pr_debug("Counter compensate: CPU%u observed %d warp\n", cpu, cur_max_warp); - /* - * Everyone initialises count in the last loop: - */ - if (i == NR_LOOPS-1) - write_c0_count(initcount); + goto retry; - atomic_inc(&count_count_stop); - while (atomic_read(&count_count_stop) != 2) - mb(); - } - /* Arrange for an interrupt in a short while */ - write_c0_compare(read_c0_count() + COUNTON); } -#undef NR_LOOPS diff --git a/arch/mips/kernel/syscall.c b/arch/mips/kernel/syscall.c index b79d13f95bf0..1bfc34a2e5b3 100644 --- a/arch/mips/kernel/syscall.c +++ b/arch/mips/kernel/syscall.c @@ -26,8 +26,10 @@ #include <linux/uaccess.h> #include <linux/slab.h> #include <linux/elf.h> +#include <linux/sched/task_stack.h> #include <asm/asm.h> +#include <asm/asm-eva.h> #include <asm/branch.h> #include <asm/cachectl.h> #include <asm/cacheflush.h> @@ -35,8 +37,9 @@ #include <asm/signal.h> #include <asm/sim.h> #include <asm/shmparam.h> +#include <asm/sync.h> #include <asm/sysmips.h> -#include <asm/uaccess.h> +#include <asm/syscalls.h> #include <asm/switch_to.h> /* @@ -60,16 +63,10 @@ SYSCALL_DEFINE6(mips_mmap, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, off_t, offset) { - unsigned long result; - - result = -EINVAL; if (offset & ~PAGE_MASK) - goto out; - - result = sys_mmap_pgoff(addr, len, prot, flags, fd, offset >> PAGE_SHIFT); - -out: - return result; + return -EINVAL; + return ksys_mmap_pgoff(addr, len, prot, flags, fd, + offset >> PAGE_SHIFT); } SYSCALL_DEFINE6(mips_mmap2, unsigned long, addr, unsigned long, len, @@ -79,11 +76,13 @@ SYSCALL_DEFINE6(mips_mmap2, unsigned long, addr, unsigned long, len, if (pgoff & (~PAGE_MASK >> 12)) return -EINVAL; - return sys_mmap_pgoff(addr, len, prot, flags, fd, pgoff >> (PAGE_SHIFT-12)); + return ksys_mmap_pgoff(addr, len, prot, flags, fd, + pgoff >> (PAGE_SHIFT - 12)); } save_static_function(sys_fork); save_static_function(sys_clone); +save_static_function(sys_clone3); SYSCALL_DEFINE1(set_thread_area, unsigned long, addr) { @@ -105,27 +104,29 @@ static inline int mips_atomic_set(unsigned long addr, unsigned long new) if (unlikely(addr & 3)) return -EINVAL; - if (unlikely(!access_ok(VERIFY_WRITE, addr, 4))) + if (unlikely(!access_ok((const void __user *)addr, 4))) return -EINVAL; - if (cpu_has_llsc && R10000_LLSC_WAR) { + if (cpu_has_llsc && IS_ENABLED(CONFIG_WAR_R10000_LLSC)) { __asm__ __volatile__ ( - " .set mips3 \n" + " .set push \n" + " .set arch=r4000 \n" " li %[err], 0 \n" "1: ll %[old], (%[addr]) \n" " move %[tmp], %[new] \n" "2: sc %[tmp], (%[addr]) \n" " beqzl %[tmp], 1b \n" "3: \n" + " .insn \n" " .section .fixup,\"ax\" \n" "4: li %[err], %[efault] \n" " j 3b \n" " .previous \n" " .section __ex_table,\"a\" \n" - " "STR(PTR)" 1b, 4b \n" - " "STR(PTR)" 2b, 4b \n" + " "STR(PTR_WD)" 1b, 4b \n" + " "STR(PTR_WD)" 2b, 4b \n" " .previous \n" - " .set mips0 \n" + " .set pop \n" : [old] "=&r" (old), [err] "=&r" (err), [tmp] "=&r" (tmp) @@ -135,26 +136,27 @@ static inline int mips_atomic_set(unsigned long addr, unsigned long new) : "memory"); } else if (cpu_has_llsc) { __asm__ __volatile__ ( - " .set mips3 \n" + " .set push \n" + " .set "MIPS_ISA_ARCH_LEVEL" \n" " li %[err], 0 \n" - "1: ll %[old], (%[addr]) \n" + "1: \n" + " " __SYNC(full, loongson3_war) " \n" + user_ll("%[old]", "(%[addr])") " move %[tmp], %[new] \n" - "2: sc %[tmp], (%[addr]) \n" - " bnez %[tmp], 4f \n" + "2: \n" + user_sc("%[tmp]", "(%[addr])") + " beqz %[tmp], 1b \n" "3: \n" - " .subsection 2 \n" - "4: b 1b \n" - " .previous \n" - " \n" + " .insn \n" " .section .fixup,\"ax\" \n" "5: li %[err], %[efault] \n" " j 3b \n" " .previous \n" " .section __ex_table,\"a\" \n" - " "STR(PTR)" 1b, 5b \n" - " "STR(PTR)" 2b, 5b \n" + " "STR(PTR_WD)" 1b, 5b \n" + " "STR(PTR_WD)" 2b, 5b \n" " .previous \n" - " .set mips0 \n" + " .set pop \n" : [old] "=&r" (old), [err] "=&r" (err), [tmp] "=&r" (tmp) @@ -197,6 +199,12 @@ static inline int mips_atomic_set(unsigned long addr, unsigned long new) unreachable(); } +/* + * mips_atomic_set() normally returns directly via syscall_exit potentially + * clobbering static registers, so be sure to preserve them. + */ +save_static_function(sys_sysmips); + SYSCALL_DEFINE3(sysmips, long, cmd, long, arg1, long, arg2) { switch (cmd) { @@ -233,12 +241,3 @@ SYSCALL_DEFINE3(cachectl, char *, addr, int, nbytes, int, op) { return -ENOSYS; } - -/* - * If we ever come here the user sp is bad. Zap the process right away. - * Due to the bad stack signaling wouldn't work. - */ -asmlinkage void bad_stack(void) -{ - do_exit(SIGSEGV); -} diff --git a/arch/mips/kernel/syscalls/Makefile b/arch/mips/kernel/syscalls/Makefile new file mode 100644 index 000000000000..56f6f093bb88 --- /dev/null +++ b/arch/mips/kernel/syscalls/Makefile @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-2.0 +kapi := arch/$(SRCARCH)/include/generated/asm +uapi := arch/$(SRCARCH)/include/generated/uapi/asm + +$(shell mkdir -p $(uapi) $(kapi)) + +syshdr := $(srctree)/scripts/syscallhdr.sh +sysnr := $(src)/syscallnr.sh +systbl := $(srctree)/scripts/syscalltbl.sh + +quiet_cmd_syshdr = SYSHDR $@ + cmd_syshdr = $(CONFIG_SHELL) $(syshdr) --offset __NR_Linux $< $@ + +quiet_cmd_sysnr = SYSNR $@ + cmd_sysnr = $(CONFIG_SHELL) '$(sysnr)' '$<' '$@' \ + '$(sysnr_abis_$(basetarget))' \ + '$(sysnr_pfx_$(basetarget))' + +quiet_cmd_systbl = SYSTBL $@ + cmd_systbl = $(CONFIG_SHELL) $(systbl) $< $@ + +$(uapi)/unistd_%.h: $(src)/syscall_%.tbl $(syshdr) FORCE + $(call if_changed,syshdr) + +sysnr_pfx_unistd_nr_n32 := N32 +sysnr_pfx_unistd_nr_n64 := 64 +sysnr_pfx_unistd_nr_o32 := O32 + +$(kapi)/unistd_nr_%.h: $(src)/syscall_%.tbl $(sysnr) FORCE + $(call if_changed,sysnr) + +$(kapi)/syscall_table_%.h: $(src)/syscall_%.tbl $(systbl) FORCE + $(call if_changed,systbl) + +uapisyshdr-y += unistd_n32.h \ + unistd_n64.h \ + unistd_o32.h +kapisyshdr-y += syscall_table_n32.h \ + syscall_table_n64.h \ + syscall_table_o32.h \ + unistd_nr_n32.h \ + unistd_nr_n64.h \ + unistd_nr_o32.h + +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) + +PHONY += all +all: $(uapisyshdr-y) $(kapisyshdr-y) + @: diff --git a/arch/mips/kernel/syscalls/syscall_n32.tbl b/arch/mips/kernel/syscalls/syscall_n32.tbl new file mode 100644 index 000000000000..8cedc83c3266 --- /dev/null +++ b/arch/mips/kernel/syscalls/syscall_n32.tbl @@ -0,0 +1,411 @@ +# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +# +# system call numbers and entry vectors for mips +# +# The format is: +# <number> <abi> <name> <entry point> <compat entry point> +# +# The <abi> is always "n32" for this file. +# +0 n32 read sys_read +1 n32 write sys_write +2 n32 open sys_open +3 n32 close sys_close +4 n32 stat sys_newstat +5 n32 fstat sys_newfstat +6 n32 lstat sys_newlstat +7 n32 poll sys_poll +8 n32 lseek sys_lseek +9 n32 mmap sys_mips_mmap +10 n32 mprotect sys_mprotect +11 n32 munmap sys_munmap +12 n32 brk sys_brk +13 n32 rt_sigaction compat_sys_rt_sigaction +14 n32 rt_sigprocmask compat_sys_rt_sigprocmask +15 n32 ioctl compat_sys_ioctl +16 n32 pread64 sys_pread64 +17 n32 pwrite64 sys_pwrite64 +18 n32 readv sys_readv +19 n32 writev sys_writev +20 n32 access sys_access +21 n32 pipe sysm_pipe +22 n32 _newselect compat_sys_select +23 n32 sched_yield sys_sched_yield +24 n32 mremap sys_mremap +25 n32 msync sys_msync +26 n32 mincore sys_mincore +27 n32 madvise sys_madvise +28 n32 shmget sys_shmget +29 n32 shmat sys_shmat +30 n32 shmctl compat_sys_old_shmctl +31 n32 dup sys_dup +32 n32 dup2 sys_dup2 +33 n32 pause sys_pause +34 n32 nanosleep sys_nanosleep_time32 +35 n32 getitimer compat_sys_getitimer +36 n32 setitimer compat_sys_setitimer +37 n32 alarm sys_alarm +38 n32 getpid sys_getpid +39 n32 sendfile compat_sys_sendfile +40 n32 socket sys_socket +41 n32 connect sys_connect +42 n32 accept sys_accept +43 n32 sendto sys_sendto +44 n32 recvfrom compat_sys_recvfrom +45 n32 sendmsg compat_sys_sendmsg +46 n32 recvmsg compat_sys_recvmsg +47 n32 shutdown sys_shutdown +48 n32 bind sys_bind +49 n32 listen sys_listen +50 n32 getsockname sys_getsockname +51 n32 getpeername sys_getpeername +52 n32 socketpair sys_socketpair +53 n32 setsockopt sys_setsockopt +54 n32 getsockopt sys_getsockopt +55 n32 clone __sys_clone +56 n32 fork __sys_fork +57 n32 execve compat_sys_execve +58 n32 exit sys_exit +59 n32 wait4 compat_sys_wait4 +60 n32 kill sys_kill +61 n32 uname sys_newuname +62 n32 semget sys_semget +63 n32 semop sys_semop +64 n32 semctl compat_sys_old_semctl +65 n32 shmdt sys_shmdt +66 n32 msgget sys_msgget +67 n32 msgsnd compat_sys_msgsnd +68 n32 msgrcv compat_sys_msgrcv +69 n32 msgctl compat_sys_old_msgctl +70 n32 fcntl compat_sys_fcntl +71 n32 flock sys_flock +72 n32 fsync sys_fsync +73 n32 fdatasync sys_fdatasync +74 n32 truncate sys_truncate +75 n32 ftruncate sys_ftruncate +76 n32 getdents compat_sys_getdents +77 n32 getcwd sys_getcwd +78 n32 chdir sys_chdir +79 n32 fchdir sys_fchdir +80 n32 rename sys_rename +81 n32 mkdir sys_mkdir +82 n32 rmdir sys_rmdir +83 n32 creat sys_creat +84 n32 link sys_link +85 n32 unlink sys_unlink +86 n32 symlink sys_symlink +87 n32 readlink sys_readlink +88 n32 chmod sys_chmod +89 n32 fchmod sys_fchmod +90 n32 chown sys_chown +91 n32 fchown sys_fchown +92 n32 lchown sys_lchown +93 n32 umask sys_umask +94 n32 gettimeofday compat_sys_gettimeofday +95 n32 getrlimit compat_sys_getrlimit +96 n32 getrusage compat_sys_getrusage +97 n32 sysinfo compat_sys_sysinfo +98 n32 times compat_sys_times +99 n32 ptrace compat_sys_ptrace +100 n32 getuid sys_getuid +101 n32 syslog sys_syslog +102 n32 getgid sys_getgid +103 n32 setuid sys_setuid +104 n32 setgid sys_setgid +105 n32 geteuid sys_geteuid +106 n32 getegid sys_getegid +107 n32 setpgid sys_setpgid +108 n32 getppid sys_getppid +109 n32 getpgrp sys_getpgrp +110 n32 setsid sys_setsid +111 n32 setreuid sys_setreuid +112 n32 setregid sys_setregid +113 n32 getgroups sys_getgroups +114 n32 setgroups sys_setgroups +115 n32 setresuid sys_setresuid +116 n32 getresuid sys_getresuid +117 n32 setresgid sys_setresgid +118 n32 getresgid sys_getresgid +119 n32 getpgid sys_getpgid +120 n32 setfsuid sys_setfsuid +121 n32 setfsgid sys_setfsgid +122 n32 getsid sys_getsid +123 n32 capget sys_capget +124 n32 capset sys_capset +125 n32 rt_sigpending compat_sys_rt_sigpending +126 n32 rt_sigtimedwait compat_sys_rt_sigtimedwait_time32 +127 n32 rt_sigqueueinfo compat_sys_rt_sigqueueinfo +128 n32 rt_sigsuspend compat_sys_rt_sigsuspend +129 n32 sigaltstack compat_sys_sigaltstack +130 n32 utime sys_utime32 +131 n32 mknod sys_mknod +132 n32 personality sys_32_personality +133 n32 ustat compat_sys_ustat +134 n32 statfs compat_sys_statfs +135 n32 fstatfs compat_sys_fstatfs +136 n32 sysfs sys_sysfs +137 n32 getpriority sys_getpriority +138 n32 setpriority sys_setpriority +139 n32 sched_setparam sys_sched_setparam +140 n32 sched_getparam sys_sched_getparam +141 n32 sched_setscheduler sys_sched_setscheduler +142 n32 sched_getscheduler sys_sched_getscheduler +143 n32 sched_get_priority_max sys_sched_get_priority_max +144 n32 sched_get_priority_min sys_sched_get_priority_min +145 n32 sched_rr_get_interval sys_sched_rr_get_interval_time32 +146 n32 mlock sys_mlock +147 n32 munlock sys_munlock +148 n32 mlockall sys_mlockall +149 n32 munlockall sys_munlockall +150 n32 vhangup sys_vhangup +151 n32 pivot_root sys_pivot_root +152 n32 _sysctl sys_ni_syscall +153 n32 prctl sys_prctl +154 n32 adjtimex sys_adjtimex_time32 +155 n32 setrlimit compat_sys_setrlimit +156 n32 chroot sys_chroot +157 n32 sync sys_sync +158 n32 acct sys_acct +159 n32 settimeofday compat_sys_settimeofday +160 n32 mount sys_mount +161 n32 umount2 sys_umount +162 n32 swapon sys_swapon +163 n32 swapoff sys_swapoff +164 n32 reboot sys_reboot +165 n32 sethostname sys_sethostname +166 n32 setdomainname sys_setdomainname +167 n32 create_module sys_ni_syscall +168 n32 init_module sys_init_module +169 n32 delete_module sys_delete_module +170 n32 get_kernel_syms sys_ni_syscall +171 n32 query_module sys_ni_syscall +172 n32 quotactl sys_quotactl +173 n32 nfsservctl sys_ni_syscall +174 n32 getpmsg sys_ni_syscall +175 n32 putpmsg sys_ni_syscall +176 n32 afs_syscall sys_ni_syscall +# 177 reserved for security +177 n32 reserved177 sys_ni_syscall +178 n32 gettid sys_gettid +179 n32 readahead sys_readahead +180 n32 setxattr sys_setxattr +181 n32 lsetxattr sys_lsetxattr +182 n32 fsetxattr sys_fsetxattr +183 n32 getxattr sys_getxattr +184 n32 lgetxattr sys_lgetxattr +185 n32 fgetxattr sys_fgetxattr +186 n32 listxattr sys_listxattr +187 n32 llistxattr sys_llistxattr +188 n32 flistxattr sys_flistxattr +189 n32 removexattr sys_removexattr +190 n32 lremovexattr sys_lremovexattr +191 n32 fremovexattr sys_fremovexattr +192 n32 tkill sys_tkill +193 n32 reserved193 sys_ni_syscall +194 n32 futex sys_futex_time32 +195 n32 sched_setaffinity compat_sys_sched_setaffinity +196 n32 sched_getaffinity compat_sys_sched_getaffinity +197 n32 cacheflush sys_cacheflush +198 n32 cachectl sys_cachectl +199 n32 sysmips __sys_sysmips +200 n32 io_setup compat_sys_io_setup +201 n32 io_destroy sys_io_destroy +202 n32 io_getevents sys_io_getevents_time32 +203 n32 io_submit compat_sys_io_submit +204 n32 io_cancel sys_io_cancel +205 n32 exit_group sys_exit_group +206 n32 lookup_dcookie sys_ni_syscall +207 n32 epoll_create sys_epoll_create +208 n32 epoll_ctl sys_epoll_ctl +209 n32 epoll_wait sys_epoll_wait +210 n32 remap_file_pages sys_remap_file_pages +211 n32 rt_sigreturn sysn32_rt_sigreturn +212 n32 fcntl64 compat_sys_fcntl64 +213 n32 set_tid_address sys_set_tid_address +214 n32 restart_syscall sys_restart_syscall +215 n32 semtimedop sys_semtimedop_time32 +216 n32 fadvise64 sys_fadvise64_64 +217 n32 statfs64 compat_sys_statfs64 +218 n32 fstatfs64 compat_sys_fstatfs64 +219 n32 sendfile64 sys_sendfile64 +220 n32 timer_create compat_sys_timer_create +221 n32 timer_settime sys_timer_settime32 +222 n32 timer_gettime sys_timer_gettime32 +223 n32 timer_getoverrun sys_timer_getoverrun +224 n32 timer_delete sys_timer_delete +225 n32 clock_settime sys_clock_settime32 +226 n32 clock_gettime sys_clock_gettime32 +227 n32 clock_getres sys_clock_getres_time32 +228 n32 clock_nanosleep sys_clock_nanosleep_time32 +229 n32 tgkill sys_tgkill +230 n32 utimes sys_utimes_time32 +231 n32 mbind sys_mbind +232 n32 get_mempolicy sys_get_mempolicy +233 n32 set_mempolicy sys_set_mempolicy +234 n32 mq_open compat_sys_mq_open +235 n32 mq_unlink sys_mq_unlink +236 n32 mq_timedsend sys_mq_timedsend_time32 +237 n32 mq_timedreceive sys_mq_timedreceive_time32 +238 n32 mq_notify compat_sys_mq_notify +239 n32 mq_getsetattr compat_sys_mq_getsetattr +240 n32 vserver sys_ni_syscall +241 n32 waitid compat_sys_waitid +# 242 was sys_setaltroot +243 n32 add_key sys_add_key +244 n32 request_key sys_request_key +245 n32 keyctl compat_sys_keyctl +246 n32 set_thread_area sys_set_thread_area +247 n32 inotify_init sys_inotify_init +248 n32 inotify_add_watch sys_inotify_add_watch +249 n32 inotify_rm_watch sys_inotify_rm_watch +250 n32 migrate_pages sys_migrate_pages +251 n32 openat sys_openat +252 n32 mkdirat sys_mkdirat +253 n32 mknodat sys_mknodat +254 n32 fchownat sys_fchownat +255 n32 futimesat sys_futimesat_time32 +256 n32 newfstatat sys_newfstatat +257 n32 unlinkat sys_unlinkat +258 n32 renameat sys_renameat +259 n32 linkat sys_linkat +260 n32 symlinkat sys_symlinkat +261 n32 readlinkat sys_readlinkat +262 n32 fchmodat sys_fchmodat +263 n32 faccessat sys_faccessat +264 n32 pselect6 compat_sys_pselect6_time32 +265 n32 ppoll compat_sys_ppoll_time32 +266 n32 unshare sys_unshare +267 n32 splice sys_splice +268 n32 sync_file_range sys_sync_file_range +269 n32 tee sys_tee +270 n32 vmsplice sys_vmsplice +271 n32 move_pages sys_move_pages +272 n32 set_robust_list compat_sys_set_robust_list +273 n32 get_robust_list compat_sys_get_robust_list +274 n32 kexec_load compat_sys_kexec_load +275 n32 getcpu sys_getcpu +276 n32 epoll_pwait compat_sys_epoll_pwait +277 n32 ioprio_set sys_ioprio_set +278 n32 ioprio_get sys_ioprio_get +279 n32 utimensat sys_utimensat_time32 +280 n32 signalfd compat_sys_signalfd +281 n32 timerfd sys_ni_syscall +282 n32 eventfd sys_eventfd +283 n32 fallocate sys_fallocate +284 n32 timerfd_create sys_timerfd_create +285 n32 timerfd_gettime sys_timerfd_gettime32 +286 n32 timerfd_settime sys_timerfd_settime32 +287 n32 signalfd4 compat_sys_signalfd4 +288 n32 eventfd2 sys_eventfd2 +289 n32 epoll_create1 sys_epoll_create1 +290 n32 dup3 sys_dup3 +291 n32 pipe2 sys_pipe2 +292 n32 inotify_init1 sys_inotify_init1 +293 n32 preadv compat_sys_preadv +294 n32 pwritev compat_sys_pwritev +295 n32 rt_tgsigqueueinfo compat_sys_rt_tgsigqueueinfo +296 n32 perf_event_open sys_perf_event_open +297 n32 accept4 sys_accept4 +298 n32 recvmmsg compat_sys_recvmmsg_time32 +299 n32 getdents64 sys_getdents64 +300 n32 fanotify_init sys_fanotify_init +301 n32 fanotify_mark sys_fanotify_mark +302 n32 prlimit64 sys_prlimit64 +303 n32 name_to_handle_at sys_name_to_handle_at +304 n32 open_by_handle_at sys_open_by_handle_at +305 n32 clock_adjtime sys_clock_adjtime32 +306 n32 syncfs sys_syncfs +307 n32 sendmmsg compat_sys_sendmmsg +308 n32 setns sys_setns +309 n32 process_vm_readv sys_process_vm_readv +310 n32 process_vm_writev sys_process_vm_writev +311 n32 kcmp sys_kcmp +312 n32 finit_module sys_finit_module +313 n32 sched_setattr sys_sched_setattr +314 n32 sched_getattr sys_sched_getattr +315 n32 renameat2 sys_renameat2 +316 n32 seccomp sys_seccomp +317 n32 getrandom sys_getrandom +318 n32 memfd_create sys_memfd_create +319 n32 bpf sys_bpf +320 n32 execveat compat_sys_execveat +321 n32 userfaultfd sys_userfaultfd +322 n32 membarrier sys_membarrier +323 n32 mlock2 sys_mlock2 +324 n32 copy_file_range sys_copy_file_range +325 n32 preadv2 compat_sys_preadv2 +326 n32 pwritev2 compat_sys_pwritev2 +327 n32 pkey_mprotect sys_pkey_mprotect +328 n32 pkey_alloc sys_pkey_alloc +329 n32 pkey_free sys_pkey_free +330 n32 statx sys_statx +331 n32 rseq sys_rseq +332 n32 io_pgetevents compat_sys_io_pgetevents +# 333 through 402 are unassigned to sync up with generic numbers +403 n32 clock_gettime64 sys_clock_gettime +404 n32 clock_settime64 sys_clock_settime +405 n32 clock_adjtime64 sys_clock_adjtime +406 n32 clock_getres_time64 sys_clock_getres +407 n32 clock_nanosleep_time64 sys_clock_nanosleep +408 n32 timer_gettime64 sys_timer_gettime +409 n32 timer_settime64 sys_timer_settime +410 n32 timerfd_gettime64 sys_timerfd_gettime +411 n32 timerfd_settime64 sys_timerfd_settime +412 n32 utimensat_time64 sys_utimensat +413 n32 pselect6_time64 compat_sys_pselect6_time64 +414 n32 ppoll_time64 compat_sys_ppoll_time64 +416 n32 io_pgetevents_time64 compat_sys_io_pgetevents_time64 +417 n32 recvmmsg_time64 compat_sys_recvmmsg_time64 +418 n32 mq_timedsend_time64 sys_mq_timedsend +419 n32 mq_timedreceive_time64 sys_mq_timedreceive +420 n32 semtimedop_time64 sys_semtimedop +421 n32 rt_sigtimedwait_time64 compat_sys_rt_sigtimedwait_time64 +422 n32 futex_time64 sys_futex +423 n32 sched_rr_get_interval_time64 sys_sched_rr_get_interval +424 n32 pidfd_send_signal sys_pidfd_send_signal +425 n32 io_uring_setup sys_io_uring_setup +426 n32 io_uring_enter sys_io_uring_enter +427 n32 io_uring_register sys_io_uring_register +428 n32 open_tree sys_open_tree +429 n32 move_mount sys_move_mount +430 n32 fsopen sys_fsopen +431 n32 fsconfig sys_fsconfig +432 n32 fsmount sys_fsmount +433 n32 fspick sys_fspick +434 n32 pidfd_open sys_pidfd_open +435 n32 clone3 __sys_clone3 +436 n32 close_range sys_close_range +437 n32 openat2 sys_openat2 +438 n32 pidfd_getfd sys_pidfd_getfd +439 n32 faccessat2 sys_faccessat2 +440 n32 process_madvise sys_process_madvise +441 n32 epoll_pwait2 compat_sys_epoll_pwait2 +442 n32 mount_setattr sys_mount_setattr +443 n32 quotactl_fd sys_quotactl_fd +444 n32 landlock_create_ruleset sys_landlock_create_ruleset +445 n32 landlock_add_rule sys_landlock_add_rule +446 n32 landlock_restrict_self sys_landlock_restrict_self +# 447 reserved for memfd_secret +448 n32 process_mrelease sys_process_mrelease +449 n32 futex_waitv sys_futex_waitv +450 n32 set_mempolicy_home_node sys_set_mempolicy_home_node +451 n32 cachestat sys_cachestat +452 n32 fchmodat2 sys_fchmodat2 +453 n32 map_shadow_stack sys_map_shadow_stack +454 n32 futex_wake sys_futex_wake +455 n32 futex_wait sys_futex_wait +456 n32 futex_requeue sys_futex_requeue +457 n32 statmount sys_statmount +458 n32 listmount sys_listmount +459 n32 lsm_get_self_attr sys_lsm_get_self_attr +460 n32 lsm_set_self_attr sys_lsm_set_self_attr +461 n32 lsm_list_modules sys_lsm_list_modules +462 n32 mseal sys_mseal +463 n32 setxattrat sys_setxattrat +464 n32 getxattrat sys_getxattrat +465 n32 listxattrat sys_listxattrat +466 n32 removexattrat sys_removexattrat +467 n32 open_tree_attr sys_open_tree_attr +468 n32 file_getattr sys_file_getattr +469 n32 file_setattr sys_file_setattr +470 n32 listns sys_listns diff --git a/arch/mips/kernel/syscalls/syscall_n64.tbl b/arch/mips/kernel/syscalls/syscall_n64.tbl new file mode 100644 index 000000000000..9b92bddf06b5 --- /dev/null +++ b/arch/mips/kernel/syscalls/syscall_n64.tbl @@ -0,0 +1,387 @@ +# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +# +# system call numbers and entry vectors for mips +# +# The format is: +# <number> <abi> <name> <entry point> +# +# The <abi> is always "n64" for this file. +# +0 n64 read sys_read +1 n64 write sys_write +2 n64 open sys_open +3 n64 close sys_close +4 n64 stat sys_newstat +5 n64 fstat sys_newfstat +6 n64 lstat sys_newlstat +7 n64 poll sys_poll +8 n64 lseek sys_lseek +9 n64 mmap sys_mips_mmap +10 n64 mprotect sys_mprotect +11 n64 munmap sys_munmap +12 n64 brk sys_brk +13 n64 rt_sigaction sys_rt_sigaction +14 n64 rt_sigprocmask sys_rt_sigprocmask +15 n64 ioctl sys_ioctl +16 n64 pread64 sys_pread64 +17 n64 pwrite64 sys_pwrite64 +18 n64 readv sys_readv +19 n64 writev sys_writev +20 n64 access sys_access +21 n64 pipe sysm_pipe +22 n64 _newselect sys_select +23 n64 sched_yield sys_sched_yield +24 n64 mremap sys_mremap +25 n64 msync sys_msync +26 n64 mincore sys_mincore +27 n64 madvise sys_madvise +28 n64 shmget sys_shmget +29 n64 shmat sys_shmat +30 n64 shmctl sys_old_shmctl +31 n64 dup sys_dup +32 n64 dup2 sys_dup2 +33 n64 pause sys_pause +34 n64 nanosleep sys_nanosleep +35 n64 getitimer sys_getitimer +36 n64 setitimer sys_setitimer +37 n64 alarm sys_alarm +38 n64 getpid sys_getpid +39 n64 sendfile sys_sendfile64 +40 n64 socket sys_socket +41 n64 connect sys_connect +42 n64 accept sys_accept +43 n64 sendto sys_sendto +44 n64 recvfrom sys_recvfrom +45 n64 sendmsg sys_sendmsg +46 n64 recvmsg sys_recvmsg +47 n64 shutdown sys_shutdown +48 n64 bind sys_bind +49 n64 listen sys_listen +50 n64 getsockname sys_getsockname +51 n64 getpeername sys_getpeername +52 n64 socketpair sys_socketpair +53 n64 setsockopt sys_setsockopt +54 n64 getsockopt sys_getsockopt +55 n64 clone __sys_clone +56 n64 fork __sys_fork +57 n64 execve sys_execve +58 n64 exit sys_exit +59 n64 wait4 sys_wait4 +60 n64 kill sys_kill +61 n64 uname sys_newuname +62 n64 semget sys_semget +63 n64 semop sys_semop +64 n64 semctl sys_old_semctl +65 n64 shmdt sys_shmdt +66 n64 msgget sys_msgget +67 n64 msgsnd sys_msgsnd +68 n64 msgrcv sys_msgrcv +69 n64 msgctl sys_old_msgctl +70 n64 fcntl sys_fcntl +71 n64 flock sys_flock +72 n64 fsync sys_fsync +73 n64 fdatasync sys_fdatasync +74 n64 truncate sys_truncate +75 n64 ftruncate sys_ftruncate +76 n64 getdents sys_getdents +77 n64 getcwd sys_getcwd +78 n64 chdir sys_chdir +79 n64 fchdir sys_fchdir +80 n64 rename sys_rename +81 n64 mkdir sys_mkdir +82 n64 rmdir sys_rmdir +83 n64 creat sys_creat +84 n64 link sys_link +85 n64 unlink sys_unlink +86 n64 symlink sys_symlink +87 n64 readlink sys_readlink +88 n64 chmod sys_chmod +89 n64 fchmod sys_fchmod +90 n64 chown sys_chown +91 n64 fchown sys_fchown +92 n64 lchown sys_lchown +93 n64 umask sys_umask +94 n64 gettimeofday sys_gettimeofday +95 n64 getrlimit sys_getrlimit +96 n64 getrusage sys_getrusage +97 n64 sysinfo sys_sysinfo +98 n64 times sys_times +99 n64 ptrace sys_ptrace +100 n64 getuid sys_getuid +101 n64 syslog sys_syslog +102 n64 getgid sys_getgid +103 n64 setuid sys_setuid +104 n64 setgid sys_setgid +105 n64 geteuid sys_geteuid +106 n64 getegid sys_getegid +107 n64 setpgid sys_setpgid +108 n64 getppid sys_getppid +109 n64 getpgrp sys_getpgrp +110 n64 setsid sys_setsid +111 n64 setreuid sys_setreuid +112 n64 setregid sys_setregid +113 n64 getgroups sys_getgroups +114 n64 setgroups sys_setgroups +115 n64 setresuid sys_setresuid +116 n64 getresuid sys_getresuid +117 n64 setresgid sys_setresgid +118 n64 getresgid sys_getresgid +119 n64 getpgid sys_getpgid +120 n64 setfsuid sys_setfsuid +121 n64 setfsgid sys_setfsgid +122 n64 getsid sys_getsid +123 n64 capget sys_capget +124 n64 capset sys_capset +125 n64 rt_sigpending sys_rt_sigpending +126 n64 rt_sigtimedwait sys_rt_sigtimedwait +127 n64 rt_sigqueueinfo sys_rt_sigqueueinfo +128 n64 rt_sigsuspend sys_rt_sigsuspend +129 n64 sigaltstack sys_sigaltstack +130 n64 utime sys_utime +131 n64 mknod sys_mknod +132 n64 personality sys_personality +133 n64 ustat sys_ustat +134 n64 statfs sys_statfs +135 n64 fstatfs sys_fstatfs +136 n64 sysfs sys_sysfs +137 n64 getpriority sys_getpriority +138 n64 setpriority sys_setpriority +139 n64 sched_setparam sys_sched_setparam +140 n64 sched_getparam sys_sched_getparam +141 n64 sched_setscheduler sys_sched_setscheduler +142 n64 sched_getscheduler sys_sched_getscheduler +143 n64 sched_get_priority_max sys_sched_get_priority_max +144 n64 sched_get_priority_min sys_sched_get_priority_min +145 n64 sched_rr_get_interval sys_sched_rr_get_interval +146 n64 mlock sys_mlock +147 n64 munlock sys_munlock +148 n64 mlockall sys_mlockall +149 n64 munlockall sys_munlockall +150 n64 vhangup sys_vhangup +151 n64 pivot_root sys_pivot_root +152 n64 _sysctl sys_ni_syscall +153 n64 prctl sys_prctl +154 n64 adjtimex sys_adjtimex +155 n64 setrlimit sys_setrlimit +156 n64 chroot sys_chroot +157 n64 sync sys_sync +158 n64 acct sys_acct +159 n64 settimeofday sys_settimeofday +160 n64 mount sys_mount +161 n64 umount2 sys_umount +162 n64 swapon sys_swapon +163 n64 swapoff sys_swapoff +164 n64 reboot sys_reboot +165 n64 sethostname sys_sethostname +166 n64 setdomainname sys_setdomainname +167 n64 create_module sys_ni_syscall +168 n64 init_module sys_init_module +169 n64 delete_module sys_delete_module +170 n64 get_kernel_syms sys_ni_syscall +171 n64 query_module sys_ni_syscall +172 n64 quotactl sys_quotactl +173 n64 nfsservctl sys_ni_syscall +174 n64 getpmsg sys_ni_syscall +175 n64 putpmsg sys_ni_syscall +176 n64 afs_syscall sys_ni_syscall +# 177 reserved for security +177 n64 reserved177 sys_ni_syscall +178 n64 gettid sys_gettid +179 n64 readahead sys_readahead +180 n64 setxattr sys_setxattr +181 n64 lsetxattr sys_lsetxattr +182 n64 fsetxattr sys_fsetxattr +183 n64 getxattr sys_getxattr +184 n64 lgetxattr sys_lgetxattr +185 n64 fgetxattr sys_fgetxattr +186 n64 listxattr sys_listxattr +187 n64 llistxattr sys_llistxattr +188 n64 flistxattr sys_flistxattr +189 n64 removexattr sys_removexattr +190 n64 lremovexattr sys_lremovexattr +191 n64 fremovexattr sys_fremovexattr +192 n64 tkill sys_tkill +193 n64 reserved193 sys_ni_syscall +194 n64 futex sys_futex +195 n64 sched_setaffinity sys_sched_setaffinity +196 n64 sched_getaffinity sys_sched_getaffinity +197 n64 cacheflush sys_cacheflush +198 n64 cachectl sys_cachectl +199 n64 sysmips __sys_sysmips +200 n64 io_setup sys_io_setup +201 n64 io_destroy sys_io_destroy +202 n64 io_getevents sys_io_getevents +203 n64 io_submit sys_io_submit +204 n64 io_cancel sys_io_cancel +205 n64 exit_group sys_exit_group +206 n64 lookup_dcookie sys_ni_syscall +207 n64 epoll_create sys_epoll_create +208 n64 epoll_ctl sys_epoll_ctl +209 n64 epoll_wait sys_epoll_wait +210 n64 remap_file_pages sys_remap_file_pages +211 n64 rt_sigreturn sys_rt_sigreturn +212 n64 set_tid_address sys_set_tid_address +213 n64 restart_syscall sys_restart_syscall +214 n64 semtimedop sys_semtimedop +215 n64 fadvise64 sys_fadvise64_64 +216 n64 timer_create sys_timer_create +217 n64 timer_settime sys_timer_settime +218 n64 timer_gettime sys_timer_gettime +219 n64 timer_getoverrun sys_timer_getoverrun +220 n64 timer_delete sys_timer_delete +221 n64 clock_settime sys_clock_settime +222 n64 clock_gettime sys_clock_gettime +223 n64 clock_getres sys_clock_getres +224 n64 clock_nanosleep sys_clock_nanosleep +225 n64 tgkill sys_tgkill +226 n64 utimes sys_utimes +227 n64 mbind sys_mbind +228 n64 get_mempolicy sys_get_mempolicy +229 n64 set_mempolicy sys_set_mempolicy +230 n64 mq_open sys_mq_open +231 n64 mq_unlink sys_mq_unlink +232 n64 mq_timedsend sys_mq_timedsend +233 n64 mq_timedreceive sys_mq_timedreceive +234 n64 mq_notify sys_mq_notify +235 n64 mq_getsetattr sys_mq_getsetattr +236 n64 vserver sys_ni_syscall +237 n64 waitid sys_waitid +# 238 was sys_setaltroot +239 n64 add_key sys_add_key +240 n64 request_key sys_request_key +241 n64 keyctl sys_keyctl +242 n64 set_thread_area sys_set_thread_area +243 n64 inotify_init sys_inotify_init +244 n64 inotify_add_watch sys_inotify_add_watch +245 n64 inotify_rm_watch sys_inotify_rm_watch +246 n64 migrate_pages sys_migrate_pages +247 n64 openat sys_openat +248 n64 mkdirat sys_mkdirat +249 n64 mknodat sys_mknodat +250 n64 fchownat sys_fchownat +251 n64 futimesat sys_futimesat +252 n64 newfstatat sys_newfstatat +253 n64 unlinkat sys_unlinkat +254 n64 renameat sys_renameat +255 n64 linkat sys_linkat +256 n64 symlinkat sys_symlinkat +257 n64 readlinkat sys_readlinkat +258 n64 fchmodat sys_fchmodat +259 n64 faccessat sys_faccessat +260 n64 pselect6 sys_pselect6 +261 n64 ppoll sys_ppoll +262 n64 unshare sys_unshare +263 n64 splice sys_splice +264 n64 sync_file_range sys_sync_file_range +265 n64 tee sys_tee +266 n64 vmsplice sys_vmsplice +267 n64 move_pages sys_move_pages +268 n64 set_robust_list sys_set_robust_list +269 n64 get_robust_list sys_get_robust_list +270 n64 kexec_load sys_kexec_load +271 n64 getcpu sys_getcpu +272 n64 epoll_pwait sys_epoll_pwait +273 n64 ioprio_set sys_ioprio_set +274 n64 ioprio_get sys_ioprio_get +275 n64 utimensat sys_utimensat +276 n64 signalfd sys_signalfd +277 n64 timerfd sys_ni_syscall +278 n64 eventfd sys_eventfd +279 n64 fallocate sys_fallocate +280 n64 timerfd_create sys_timerfd_create +281 n64 timerfd_gettime sys_timerfd_gettime +282 n64 timerfd_settime sys_timerfd_settime +283 n64 signalfd4 sys_signalfd4 +284 n64 eventfd2 sys_eventfd2 +285 n64 epoll_create1 sys_epoll_create1 +286 n64 dup3 sys_dup3 +287 n64 pipe2 sys_pipe2 +288 n64 inotify_init1 sys_inotify_init1 +289 n64 preadv sys_preadv +290 n64 pwritev sys_pwritev +291 n64 rt_tgsigqueueinfo sys_rt_tgsigqueueinfo +292 n64 perf_event_open sys_perf_event_open +293 n64 accept4 sys_accept4 +294 n64 recvmmsg sys_recvmmsg +295 n64 fanotify_init sys_fanotify_init +296 n64 fanotify_mark sys_fanotify_mark +297 n64 prlimit64 sys_prlimit64 +298 n64 name_to_handle_at sys_name_to_handle_at +299 n64 open_by_handle_at sys_open_by_handle_at +300 n64 clock_adjtime sys_clock_adjtime +301 n64 syncfs sys_syncfs +302 n64 sendmmsg sys_sendmmsg +303 n64 setns sys_setns +304 n64 process_vm_readv sys_process_vm_readv +305 n64 process_vm_writev sys_process_vm_writev +306 n64 kcmp sys_kcmp +307 n64 finit_module sys_finit_module +308 n64 getdents64 sys_getdents64 +309 n64 sched_setattr sys_sched_setattr +310 n64 sched_getattr sys_sched_getattr +311 n64 renameat2 sys_renameat2 +312 n64 seccomp sys_seccomp +313 n64 getrandom sys_getrandom +314 n64 memfd_create sys_memfd_create +315 n64 bpf sys_bpf +316 n64 execveat sys_execveat +317 n64 userfaultfd sys_userfaultfd +318 n64 membarrier sys_membarrier +319 n64 mlock2 sys_mlock2 +320 n64 copy_file_range sys_copy_file_range +321 n64 preadv2 sys_preadv2 +322 n64 pwritev2 sys_pwritev2 +323 n64 pkey_mprotect sys_pkey_mprotect +324 n64 pkey_alloc sys_pkey_alloc +325 n64 pkey_free sys_pkey_free +326 n64 statx sys_statx +327 n64 rseq sys_rseq +328 n64 io_pgetevents sys_io_pgetevents +# 329 through 423 are reserved to sync up with other architectures +424 n64 pidfd_send_signal sys_pidfd_send_signal +425 n64 io_uring_setup sys_io_uring_setup +426 n64 io_uring_enter sys_io_uring_enter +427 n64 io_uring_register sys_io_uring_register +428 n64 open_tree sys_open_tree +429 n64 move_mount sys_move_mount +430 n64 fsopen sys_fsopen +431 n64 fsconfig sys_fsconfig +432 n64 fsmount sys_fsmount +433 n64 fspick sys_fspick +434 n64 pidfd_open sys_pidfd_open +435 n64 clone3 __sys_clone3 +436 n64 close_range sys_close_range +437 n64 openat2 sys_openat2 +438 n64 pidfd_getfd sys_pidfd_getfd +439 n64 faccessat2 sys_faccessat2 +440 n64 process_madvise sys_process_madvise +441 n64 epoll_pwait2 sys_epoll_pwait2 +442 n64 mount_setattr sys_mount_setattr +443 n64 quotactl_fd sys_quotactl_fd +444 n64 landlock_create_ruleset sys_landlock_create_ruleset +445 n64 landlock_add_rule sys_landlock_add_rule +446 n64 landlock_restrict_self sys_landlock_restrict_self +# 447 reserved for memfd_secret +448 n64 process_mrelease sys_process_mrelease +449 n64 futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 n64 cachestat sys_cachestat +452 n64 fchmodat2 sys_fchmodat2 +453 n64 map_shadow_stack sys_map_shadow_stack +454 n64 futex_wake sys_futex_wake +455 n64 futex_wait sys_futex_wait +456 n64 futex_requeue sys_futex_requeue +457 n64 statmount sys_statmount +458 n64 listmount sys_listmount +459 n64 lsm_get_self_attr sys_lsm_get_self_attr +460 n64 lsm_set_self_attr sys_lsm_set_self_attr +461 n64 lsm_list_modules sys_lsm_list_modules +462 n64 mseal sys_mseal +463 n64 setxattrat sys_setxattrat +464 n64 getxattrat sys_getxattrat +465 n64 listxattrat sys_listxattrat +466 n64 removexattrat sys_removexattrat +467 n64 open_tree_attr sys_open_tree_attr +468 n64 file_getattr sys_file_getattr +469 n64 file_setattr sys_file_setattr +470 n64 listns sys_listns diff --git a/arch/mips/kernel/syscalls/syscall_o32.tbl b/arch/mips/kernel/syscalls/syscall_o32.tbl new file mode 100644 index 000000000000..f810b8a55716 --- /dev/null +++ b/arch/mips/kernel/syscalls/syscall_o32.tbl @@ -0,0 +1,460 @@ +# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +# +# system call numbers and entry vectors for mips +# +# The format is: +# <number> <abi> <name> <entry point> <compat entry point> +# +# The <abi> is always "o32" for this file. +# +0 o32 syscall sys_syscall sys32_syscall +1 o32 exit sys_exit +2 o32 fork __sys_fork +3 o32 read sys_read +4 o32 write sys_write +5 o32 open sys_open compat_sys_open +6 o32 close sys_close +7 o32 waitpid sys_waitpid +8 o32 creat sys_creat +9 o32 link sys_link +10 o32 unlink sys_unlink +11 o32 execve sys_execve compat_sys_execve +12 o32 chdir sys_chdir +13 o32 time sys_time32 +14 o32 mknod sys_mknod +15 o32 chmod sys_chmod +16 o32 lchown sys_lchown +17 o32 break sys_ni_syscall +# 18 was sys_stat +18 o32 unused18 sys_ni_syscall +19 o32 lseek sys_lseek compat_sys_lseek +20 o32 getpid sys_getpid +21 o32 mount sys_mount +22 o32 umount sys_oldumount +23 o32 setuid sys_setuid +24 o32 getuid sys_getuid +25 o32 stime sys_stime32 +26 o32 ptrace sys_ptrace compat_sys_ptrace +27 o32 alarm sys_alarm +# 28 was sys_fstat +28 o32 unused28 sys_ni_syscall +29 o32 pause sys_pause +30 o32 utime sys_utime32 +31 o32 stty sys_ni_syscall +32 o32 gtty sys_ni_syscall +33 o32 access sys_access +34 o32 nice sys_nice +35 o32 ftime sys_ni_syscall +36 o32 sync sys_sync +37 o32 kill sys_kill +38 o32 rename sys_rename +39 o32 mkdir sys_mkdir +40 o32 rmdir sys_rmdir +41 o32 dup sys_dup +42 o32 pipe sysm_pipe +43 o32 times sys_times compat_sys_times +44 o32 prof sys_ni_syscall +45 o32 brk sys_brk +46 o32 setgid sys_setgid +47 o32 getgid sys_getgid +48 o32 signal sys_ni_syscall +49 o32 geteuid sys_geteuid +50 o32 getegid sys_getegid +51 o32 acct sys_acct +52 o32 umount2 sys_umount +53 o32 lock sys_ni_syscall +54 o32 ioctl sys_ioctl compat_sys_ioctl +55 o32 fcntl sys_fcntl compat_sys_fcntl +56 o32 mpx sys_ni_syscall +57 o32 setpgid sys_setpgid +58 o32 ulimit sys_ni_syscall +59 o32 unused59 sys_olduname +60 o32 umask sys_umask +61 o32 chroot sys_chroot +62 o32 ustat sys_ustat compat_sys_ustat +63 o32 dup2 sys_dup2 +64 o32 getppid sys_getppid +65 o32 getpgrp sys_getpgrp +66 o32 setsid sys_setsid +67 o32 sigaction sys_sigaction sys_32_sigaction +68 o32 sgetmask sys_sgetmask +69 o32 ssetmask sys_ssetmask +70 o32 setreuid sys_setreuid +71 o32 setregid sys_setregid +72 o32 sigsuspend sys_sigsuspend sys32_sigsuspend +73 o32 sigpending sys_sigpending compat_sys_sigpending +74 o32 sethostname sys_sethostname +75 o32 setrlimit sys_setrlimit compat_sys_setrlimit +76 o32 getrlimit sys_getrlimit compat_sys_getrlimit +77 o32 getrusage sys_getrusage compat_sys_getrusage +78 o32 gettimeofday sys_gettimeofday compat_sys_gettimeofday +79 o32 settimeofday sys_settimeofday compat_sys_settimeofday +80 o32 getgroups sys_getgroups +81 o32 setgroups sys_setgroups +# 82 was old_select +82 o32 reserved82 sys_ni_syscall +83 o32 symlink sys_symlink +# 84 was sys_lstat +84 o32 unused84 sys_ni_syscall +85 o32 readlink sys_readlink +86 o32 uselib sys_uselib +87 o32 swapon sys_swapon +88 o32 reboot sys_reboot +89 o32 readdir sys_old_readdir compat_sys_old_readdir +90 o32 mmap sys_mips_mmap +91 o32 munmap sys_munmap +92 o32 truncate sys_truncate compat_sys_truncate +93 o32 ftruncate sys_ftruncate compat_sys_ftruncate +94 o32 fchmod sys_fchmod +95 o32 fchown sys_fchown +96 o32 getpriority sys_getpriority +97 o32 setpriority sys_setpriority +98 o32 profil sys_ni_syscall +99 o32 statfs sys_statfs compat_sys_statfs +100 o32 fstatfs sys_fstatfs compat_sys_fstatfs +101 o32 ioperm sys_ni_syscall +102 o32 socketcall sys_socketcall compat_sys_socketcall +103 o32 syslog sys_syslog +104 o32 setitimer sys_setitimer compat_sys_setitimer +105 o32 getitimer sys_getitimer compat_sys_getitimer +106 o32 stat sys_newstat compat_sys_newstat +107 o32 lstat sys_newlstat compat_sys_newlstat +108 o32 fstat sys_newfstat compat_sys_newfstat +109 o32 unused109 sys_uname +110 o32 iopl sys_ni_syscall +111 o32 vhangup sys_vhangup +112 o32 idle sys_ni_syscall +113 o32 vm86 sys_ni_syscall +114 o32 wait4 sys_wait4 compat_sys_wait4 +115 o32 swapoff sys_swapoff +116 o32 sysinfo sys_sysinfo compat_sys_sysinfo +117 o32 ipc sys_ipc compat_sys_ipc +118 o32 fsync sys_fsync +119 o32 sigreturn sys_sigreturn sys32_sigreturn +120 o32 clone __sys_clone +121 o32 setdomainname sys_setdomainname +122 o32 uname sys_newuname +123 o32 modify_ldt sys_ni_syscall +124 o32 adjtimex sys_adjtimex_time32 +125 o32 mprotect sys_mprotect +126 o32 sigprocmask sys_sigprocmask compat_sys_sigprocmask +127 o32 create_module sys_ni_syscall +128 o32 init_module sys_init_module +129 o32 delete_module sys_delete_module +130 o32 get_kernel_syms sys_ni_syscall +131 o32 quotactl sys_quotactl +132 o32 getpgid sys_getpgid +133 o32 fchdir sys_fchdir +134 o32 bdflush sys_ni_syscall +135 o32 sysfs sys_sysfs +136 o32 personality sys_personality sys_32_personality +137 o32 afs_syscall sys_ni_syscall +138 o32 setfsuid sys_setfsuid +139 o32 setfsgid sys_setfsgid +140 o32 _llseek sys_llseek sys_32_llseek +141 o32 getdents sys_getdents compat_sys_getdents +142 o32 _newselect sys_select compat_sys_select +143 o32 flock sys_flock +144 o32 msync sys_msync +145 o32 readv sys_readv +146 o32 writev sys_writev +147 o32 cacheflush sys_cacheflush +148 o32 cachectl sys_cachectl +149 o32 sysmips __sys_sysmips +150 o32 unused150 sys_ni_syscall +151 o32 getsid sys_getsid +152 o32 fdatasync sys_fdatasync +153 o32 _sysctl sys_ni_syscall +154 o32 mlock sys_mlock +155 o32 munlock sys_munlock +156 o32 mlockall sys_mlockall +157 o32 munlockall sys_munlockall +158 o32 sched_setparam sys_sched_setparam +159 o32 sched_getparam sys_sched_getparam +160 o32 sched_setscheduler sys_sched_setscheduler +161 o32 sched_getscheduler sys_sched_getscheduler +162 o32 sched_yield sys_sched_yield +163 o32 sched_get_priority_max sys_sched_get_priority_max +164 o32 sched_get_priority_min sys_sched_get_priority_min +165 o32 sched_rr_get_interval sys_sched_rr_get_interval_time32 +166 o32 nanosleep sys_nanosleep_time32 +167 o32 mremap sys_mremap +168 o32 accept sys_accept +169 o32 bind sys_bind +170 o32 connect sys_connect +171 o32 getpeername sys_getpeername +172 o32 getsockname sys_getsockname +173 o32 getsockopt sys_getsockopt sys_getsockopt +174 o32 listen sys_listen +175 o32 recv sys_recv compat_sys_recv +176 o32 recvfrom sys_recvfrom compat_sys_recvfrom +177 o32 recvmsg sys_recvmsg compat_sys_recvmsg +178 o32 send sys_send +179 o32 sendmsg sys_sendmsg compat_sys_sendmsg +180 o32 sendto sys_sendto +181 o32 setsockopt sys_setsockopt sys_setsockopt +182 o32 shutdown sys_shutdown +183 o32 socket sys_socket +184 o32 socketpair sys_socketpair +185 o32 setresuid sys_setresuid +186 o32 getresuid sys_getresuid +187 o32 query_module sys_ni_syscall +188 o32 poll sys_poll +189 o32 nfsservctl sys_ni_syscall +190 o32 setresgid sys_setresgid +191 o32 getresgid sys_getresgid +192 o32 prctl sys_prctl +193 o32 rt_sigreturn sys_rt_sigreturn sys32_rt_sigreturn +194 o32 rt_sigaction sys_rt_sigaction compat_sys_rt_sigaction +195 o32 rt_sigprocmask sys_rt_sigprocmask compat_sys_rt_sigprocmask +196 o32 rt_sigpending sys_rt_sigpending compat_sys_rt_sigpending +197 o32 rt_sigtimedwait sys_rt_sigtimedwait_time32 compat_sys_rt_sigtimedwait_time32 +198 o32 rt_sigqueueinfo sys_rt_sigqueueinfo compat_sys_rt_sigqueueinfo +199 o32 rt_sigsuspend sys_rt_sigsuspend compat_sys_rt_sigsuspend +200 o32 pread64 sys_pread64 sys_32_pread +201 o32 pwrite64 sys_pwrite64 sys_32_pwrite +202 o32 chown sys_chown +203 o32 getcwd sys_getcwd +204 o32 capget sys_capget +205 o32 capset sys_capset +206 o32 sigaltstack sys_sigaltstack compat_sys_sigaltstack +207 o32 sendfile sys_sendfile compat_sys_sendfile +208 o32 getpmsg sys_ni_syscall +209 o32 putpmsg sys_ni_syscall +210 o32 mmap2 sys_mips_mmap2 +211 o32 truncate64 sys_truncate64 sys_32_truncate64 +212 o32 ftruncate64 sys_ftruncate64 sys_32_ftruncate64 +213 o32 stat64 sys_stat64 sys_newstat +214 o32 lstat64 sys_lstat64 sys_newlstat +215 o32 fstat64 sys_fstat64 sys_newfstat +216 o32 pivot_root sys_pivot_root +217 o32 mincore sys_mincore +218 o32 madvise sys_madvise +219 o32 getdents64 sys_getdents64 +220 o32 fcntl64 sys_fcntl64 compat_sys_fcntl64 +221 o32 reserved221 sys_ni_syscall +222 o32 gettid sys_gettid +223 o32 readahead sys_readahead sys32_readahead +224 o32 setxattr sys_setxattr +225 o32 lsetxattr sys_lsetxattr +226 o32 fsetxattr sys_fsetxattr +227 o32 getxattr sys_getxattr +228 o32 lgetxattr sys_lgetxattr +229 o32 fgetxattr sys_fgetxattr +230 o32 listxattr sys_listxattr +231 o32 llistxattr sys_llistxattr +232 o32 flistxattr sys_flistxattr +233 o32 removexattr sys_removexattr +234 o32 lremovexattr sys_lremovexattr +235 o32 fremovexattr sys_fremovexattr +236 o32 tkill sys_tkill +237 o32 sendfile64 sys_sendfile64 +238 o32 futex sys_futex_time32 +239 o32 sched_setaffinity sys_sched_setaffinity compat_sys_sched_setaffinity +240 o32 sched_getaffinity sys_sched_getaffinity compat_sys_sched_getaffinity +241 o32 io_setup sys_io_setup compat_sys_io_setup +242 o32 io_destroy sys_io_destroy +243 o32 io_getevents sys_io_getevents_time32 +244 o32 io_submit sys_io_submit compat_sys_io_submit +245 o32 io_cancel sys_io_cancel +246 o32 exit_group sys_exit_group +247 o32 lookup_dcookie sys_ni_syscall +248 o32 epoll_create sys_epoll_create +249 o32 epoll_ctl sys_epoll_ctl +250 o32 epoll_wait sys_epoll_wait +251 o32 remap_file_pages sys_remap_file_pages +252 o32 set_tid_address sys_set_tid_address +253 o32 restart_syscall sys_restart_syscall +254 o32 fadvise64 sys_fadvise64_64 sys32_fadvise64_64 +255 o32 statfs64 sys_statfs64 compat_sys_statfs64 +256 o32 fstatfs64 sys_fstatfs64 compat_sys_fstatfs64 +257 o32 timer_create sys_timer_create compat_sys_timer_create +258 o32 timer_settime sys_timer_settime32 +259 o32 timer_gettime sys_timer_gettime32 +260 o32 timer_getoverrun sys_timer_getoverrun +261 o32 timer_delete sys_timer_delete +262 o32 clock_settime sys_clock_settime32 +263 o32 clock_gettime sys_clock_gettime32 +264 o32 clock_getres sys_clock_getres_time32 +265 o32 clock_nanosleep sys_clock_nanosleep_time32 +266 o32 tgkill sys_tgkill +267 o32 utimes sys_utimes_time32 +268 o32 mbind sys_mbind +269 o32 get_mempolicy sys_get_mempolicy +270 o32 set_mempolicy sys_set_mempolicy +271 o32 mq_open sys_mq_open compat_sys_mq_open +272 o32 mq_unlink sys_mq_unlink +273 o32 mq_timedsend sys_mq_timedsend_time32 +274 o32 mq_timedreceive sys_mq_timedreceive_time32 +275 o32 mq_notify sys_mq_notify compat_sys_mq_notify +276 o32 mq_getsetattr sys_mq_getsetattr compat_sys_mq_getsetattr +277 o32 vserver sys_ni_syscall +278 o32 waitid sys_waitid compat_sys_waitid +# 279 was sys_setaltroot +280 o32 add_key sys_add_key +281 o32 request_key sys_request_key +282 o32 keyctl sys_keyctl compat_sys_keyctl +283 o32 set_thread_area sys_set_thread_area +284 o32 inotify_init sys_inotify_init +285 o32 inotify_add_watch sys_inotify_add_watch +286 o32 inotify_rm_watch sys_inotify_rm_watch +287 o32 migrate_pages sys_migrate_pages +288 o32 openat sys_openat compat_sys_openat +289 o32 mkdirat sys_mkdirat +290 o32 mknodat sys_mknodat +291 o32 fchownat sys_fchownat +292 o32 futimesat sys_futimesat_time32 +293 o32 fstatat64 sys_fstatat64 sys_newfstatat +294 o32 unlinkat sys_unlinkat +295 o32 renameat sys_renameat +296 o32 linkat sys_linkat +297 o32 symlinkat sys_symlinkat +298 o32 readlinkat sys_readlinkat +299 o32 fchmodat sys_fchmodat +300 o32 faccessat sys_faccessat +301 o32 pselect6 sys_pselect6_time32 compat_sys_pselect6_time32 +302 o32 ppoll sys_ppoll_time32 compat_sys_ppoll_time32 +303 o32 unshare sys_unshare +304 o32 splice sys_splice +305 o32 sync_file_range sys_sync_file_range sys32_sync_file_range +306 o32 tee sys_tee +307 o32 vmsplice sys_vmsplice +308 o32 move_pages sys_move_pages +309 o32 set_robust_list sys_set_robust_list compat_sys_set_robust_list +310 o32 get_robust_list sys_get_robust_list compat_sys_get_robust_list +311 o32 kexec_load sys_kexec_load compat_sys_kexec_load +312 o32 getcpu sys_getcpu +313 o32 epoll_pwait sys_epoll_pwait compat_sys_epoll_pwait +314 o32 ioprio_set sys_ioprio_set +315 o32 ioprio_get sys_ioprio_get +316 o32 utimensat sys_utimensat_time32 +317 o32 signalfd sys_signalfd compat_sys_signalfd +318 o32 timerfd sys_ni_syscall +319 o32 eventfd sys_eventfd +320 o32 fallocate sys_fallocate sys32_fallocate +321 o32 timerfd_create sys_timerfd_create +322 o32 timerfd_gettime sys_timerfd_gettime32 +323 o32 timerfd_settime sys_timerfd_settime32 +324 o32 signalfd4 sys_signalfd4 compat_sys_signalfd4 +325 o32 eventfd2 sys_eventfd2 +326 o32 epoll_create1 sys_epoll_create1 +327 o32 dup3 sys_dup3 +328 o32 pipe2 sys_pipe2 +329 o32 inotify_init1 sys_inotify_init1 +330 o32 preadv sys_preadv compat_sys_preadv +331 o32 pwritev sys_pwritev compat_sys_pwritev +332 o32 rt_tgsigqueueinfo sys_rt_tgsigqueueinfo compat_sys_rt_tgsigqueueinfo +333 o32 perf_event_open sys_perf_event_open +334 o32 accept4 sys_accept4 +335 o32 recvmmsg sys_recvmmsg_time32 compat_sys_recvmmsg_time32 +336 o32 fanotify_init sys_fanotify_init +337 o32 fanotify_mark sys_fanotify_mark compat_sys_fanotify_mark +338 o32 prlimit64 sys_prlimit64 +339 o32 name_to_handle_at sys_name_to_handle_at +340 o32 open_by_handle_at sys_open_by_handle_at compat_sys_open_by_handle_at +341 o32 clock_adjtime sys_clock_adjtime32 +342 o32 syncfs sys_syncfs +343 o32 sendmmsg sys_sendmmsg compat_sys_sendmmsg +344 o32 setns sys_setns +345 o32 process_vm_readv sys_process_vm_readv +346 o32 process_vm_writev sys_process_vm_writev +347 o32 kcmp sys_kcmp +348 o32 finit_module sys_finit_module +349 o32 sched_setattr sys_sched_setattr +350 o32 sched_getattr sys_sched_getattr +351 o32 renameat2 sys_renameat2 +352 o32 seccomp sys_seccomp +353 o32 getrandom sys_getrandom +354 o32 memfd_create sys_memfd_create +355 o32 bpf sys_bpf +356 o32 execveat sys_execveat compat_sys_execveat +357 o32 userfaultfd sys_userfaultfd +358 o32 membarrier sys_membarrier +359 o32 mlock2 sys_mlock2 +360 o32 copy_file_range sys_copy_file_range +361 o32 preadv2 sys_preadv2 compat_sys_preadv2 +362 o32 pwritev2 sys_pwritev2 compat_sys_pwritev2 +363 o32 pkey_mprotect sys_pkey_mprotect +364 o32 pkey_alloc sys_pkey_alloc +365 o32 pkey_free sys_pkey_free +366 o32 statx sys_statx +367 o32 rseq sys_rseq +368 o32 io_pgetevents sys_io_pgetevents_time32 compat_sys_io_pgetevents +# room for arch specific calls +393 o32 semget sys_semget +394 o32 semctl sys_semctl compat_sys_semctl +395 o32 shmget sys_shmget +396 o32 shmctl sys_shmctl compat_sys_shmctl +397 o32 shmat sys_shmat compat_sys_shmat +398 o32 shmdt sys_shmdt +399 o32 msgget sys_msgget +400 o32 msgsnd sys_msgsnd compat_sys_msgsnd +401 o32 msgrcv sys_msgrcv compat_sys_msgrcv +402 o32 msgctl sys_msgctl compat_sys_msgctl +403 o32 clock_gettime64 sys_clock_gettime sys_clock_gettime +404 o32 clock_settime64 sys_clock_settime sys_clock_settime +405 o32 clock_adjtime64 sys_clock_adjtime sys_clock_adjtime +406 o32 clock_getres_time64 sys_clock_getres sys_clock_getres +407 o32 clock_nanosleep_time64 sys_clock_nanosleep sys_clock_nanosleep +408 o32 timer_gettime64 sys_timer_gettime sys_timer_gettime +409 o32 timer_settime64 sys_timer_settime sys_timer_settime +410 o32 timerfd_gettime64 sys_timerfd_gettime sys_timerfd_gettime +411 o32 timerfd_settime64 sys_timerfd_settime sys_timerfd_settime +412 o32 utimensat_time64 sys_utimensat sys_utimensat +413 o32 pselect6_time64 sys_pselect6 compat_sys_pselect6_time64 +414 o32 ppoll_time64 sys_ppoll compat_sys_ppoll_time64 +416 o32 io_pgetevents_time64 sys_io_pgetevents compat_sys_io_pgetevents_time64 +417 o32 recvmmsg_time64 sys_recvmmsg compat_sys_recvmmsg_time64 +418 o32 mq_timedsend_time64 sys_mq_timedsend sys_mq_timedsend +419 o32 mq_timedreceive_time64 sys_mq_timedreceive sys_mq_timedreceive +420 o32 semtimedop_time64 sys_semtimedop sys_semtimedop +421 o32 rt_sigtimedwait_time64 sys_rt_sigtimedwait compat_sys_rt_sigtimedwait_time64 +422 o32 futex_time64 sys_futex sys_futex +423 o32 sched_rr_get_interval_time64 sys_sched_rr_get_interval sys_sched_rr_get_interval +424 o32 pidfd_send_signal sys_pidfd_send_signal +425 o32 io_uring_setup sys_io_uring_setup +426 o32 io_uring_enter sys_io_uring_enter +427 o32 io_uring_register sys_io_uring_register +428 o32 open_tree sys_open_tree +429 o32 move_mount sys_move_mount +430 o32 fsopen sys_fsopen +431 o32 fsconfig sys_fsconfig +432 o32 fsmount sys_fsmount +433 o32 fspick sys_fspick +434 o32 pidfd_open sys_pidfd_open +435 o32 clone3 __sys_clone3 +436 o32 close_range sys_close_range +437 o32 openat2 sys_openat2 +438 o32 pidfd_getfd sys_pidfd_getfd +439 o32 faccessat2 sys_faccessat2 +440 o32 process_madvise sys_process_madvise +441 o32 epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 o32 mount_setattr sys_mount_setattr +443 o32 quotactl_fd sys_quotactl_fd +444 o32 landlock_create_ruleset sys_landlock_create_ruleset +445 o32 landlock_add_rule sys_landlock_add_rule +446 o32 landlock_restrict_self sys_landlock_restrict_self +# 447 reserved for memfd_secret +448 o32 process_mrelease sys_process_mrelease +449 o32 futex_waitv sys_futex_waitv +450 o32 set_mempolicy_home_node sys_set_mempolicy_home_node +451 o32 cachestat sys_cachestat +452 o32 fchmodat2 sys_fchmodat2 +453 o32 map_shadow_stack sys_map_shadow_stack +454 o32 futex_wake sys_futex_wake +455 o32 futex_wait sys_futex_wait +456 o32 futex_requeue sys_futex_requeue +457 o32 statmount sys_statmount +458 o32 listmount sys_listmount +459 o32 lsm_get_self_attr sys_lsm_get_self_attr +460 o32 lsm_set_self_attr sys_lsm_set_self_attr +461 o32 lsm_list_modules sys_lsm_list_modules +462 o32 mseal sys_mseal +463 o32 setxattrat sys_setxattrat +464 o32 getxattrat sys_getxattrat +465 o32 listxattrat sys_listxattrat +466 o32 removexattrat sys_removexattrat +467 o32 open_tree_attr sys_open_tree_attr +468 o32 file_getattr sys_file_getattr +469 o32 file_setattr sys_file_setattr +470 o32 listns sys_listns diff --git a/arch/mips/kernel/syscalls/syscallnr.sh b/arch/mips/kernel/syscalls/syscallnr.sh new file mode 100644 index 000000000000..c190bbefbfc2 --- /dev/null +++ b/arch/mips/kernel/syscalls/syscallnr.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +in="$1" +out="$2" +my_abis=`echo "($3)" | tr ',' '|'` +prefix="$4" + +fileguard=_UAPI_ASM_MIPS_`basename "$out" | sed \ + -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/' \ + -e 's/[^A-Z0-9_]/_/g' -e 's/__/_/g'` +grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( + printf "#ifndef %s\n" "${fileguard}" + printf "#define %s\n" "${fileguard}" + printf "\n" + + nxt=0 + while read nr abi name entry compat ; do + nxt=$((nr+1)) + done + + printf "#define __NR_%s_Linux_syscalls\t%s\n" "${prefix}" "${nxt}" + printf "\n" + printf "#endif /* %s */" "${fileguard}" + printf "\n" +) > "$out" diff --git a/arch/mips/kernel/sysrq.c b/arch/mips/kernel/sysrq.c new file mode 100644 index 000000000000..2e98049fe783 --- /dev/null +++ b/arch/mips/kernel/sysrq.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * MIPS specific sysrq operations. + * + * Copyright (C) 2015 Imagination Technologies Ltd. + */ +#include <linux/init.h> +#include <linux/smp.h> +#include <linux/spinlock.h> +#include <linux/sysrq.h> +#include <linux/workqueue.h> + +#include <asm/cpu-features.h> +#include <asm/mipsregs.h> +#include <asm/tlbdebug.h> + +/* + * Dump TLB entries on all CPUs. + */ + +static DEFINE_SPINLOCK(show_lock); + +static void sysrq_tlbdump_single(void *dummy) +{ + unsigned long flags; + + spin_lock_irqsave(&show_lock, flags); + + pr_info("CPU%d:\n", smp_processor_id()); + dump_tlb_regs(); + pr_info("\n"); + dump_tlb_all(); + pr_info("\n"); + + spin_unlock_irqrestore(&show_lock, flags); +} + +#ifdef CONFIG_SMP +static void sysrq_tlbdump_othercpus(struct work_struct *dummy) +{ + smp_call_function(sysrq_tlbdump_single, NULL, 0); +} + +static DECLARE_WORK(sysrq_tlbdump, sysrq_tlbdump_othercpus); +#endif + +static void sysrq_handle_tlbdump(u8 key) +{ + sysrq_tlbdump_single(NULL); +#ifdef CONFIG_SMP + schedule_work(&sysrq_tlbdump); +#endif +} + +static const struct sysrq_key_op sysrq_tlbdump_op = { + .handler = sysrq_handle_tlbdump, + .help_msg = "show-tlbs(x)", + .action_msg = "Show TLB entries", + .enable_mask = SYSRQ_ENABLE_DUMP, +}; + +static int __init mips_sysrq_init(void) +{ + return register_sysrq_key('x', &sysrq_tlbdump_op); +} +arch_initcall(mips_sysrq_init); diff --git a/arch/mips/kernel/time.c b/arch/mips/kernel/time.c index 9d686bf97b0e..ed339d7979f3 100644 --- a/arch/mips/kernel/time.c +++ b/arch/mips/kernel/time.c @@ -1,14 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright 2001 MontaVista Software Inc. * Author: Jun Sun, jsun@mvista.com or jsun@junsun.net * Copyright (c) 2003, 2004 Maciej W. Rozycki * * Common time service routines for MIPS machines. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. */ #include <linux/bug.h> #include <linux/clockchips.h> @@ -22,32 +18,87 @@ #include <linux/smp.h> #include <linux/spinlock.h> #include <linux/export.h> +#include <linux/cpufreq.h> +#include <linux/delay.h> #include <asm/cpu-features.h> +#include <asm/cpu-type.h> #include <asm/div64.h> -#include <asm/smtc_ipi.h> #include <asm/time.h> -/* - * forward reference - */ -DEFINE_SPINLOCK(rtc_lock); -EXPORT_SYMBOL(rtc_lock); +#ifdef CONFIG_CPU_FREQ -int __weak rtc_mips_set_time(unsigned long sec) -{ - return 0; -} +static DEFINE_PER_CPU(unsigned long, pcp_lpj_ref); +static DEFINE_PER_CPU(unsigned long, pcp_lpj_ref_freq); +static unsigned long glb_lpj_ref; +static unsigned long glb_lpj_ref_freq; -int __weak rtc_mips_set_mmss(unsigned long nowtime) +static int cpufreq_callback(struct notifier_block *nb, + unsigned long val, void *data) { - return rtc_mips_set_time(nowtime); + struct cpufreq_freqs *freq = data; + struct cpumask *cpus = freq->policy->cpus; + unsigned long lpj; + int cpu; + + /* + * Skip lpj numbers adjustment if the CPU-freq transition is safe for + * the loops delay. (Is this possible?) + */ + if (freq->flags & CPUFREQ_CONST_LOOPS) + return NOTIFY_OK; + + /* Save the initial values of the lpjes for future scaling. */ + if (!glb_lpj_ref) { + glb_lpj_ref = boot_cpu_data.udelay_val; + glb_lpj_ref_freq = freq->old; + + for_each_online_cpu(cpu) { + per_cpu(pcp_lpj_ref, cpu) = + cpu_data[cpu].udelay_val; + per_cpu(pcp_lpj_ref_freq, cpu) = freq->old; + } + } + + /* + * Adjust global lpj variable and per-CPU udelay_val number in + * accordance with the new CPU frequency. + */ + if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || + (val == CPUFREQ_POSTCHANGE && freq->old > freq->new)) { + loops_per_jiffy = cpufreq_scale(glb_lpj_ref, + glb_lpj_ref_freq, + freq->new); + + for_each_cpu(cpu, cpus) { + lpj = cpufreq_scale(per_cpu(pcp_lpj_ref, cpu), + per_cpu(pcp_lpj_ref_freq, cpu), + freq->new); + cpu_data[cpu].udelay_val = (unsigned int)lpj; + } + } + + return NOTIFY_OK; } -int update_persistent_clock(struct timespec now) +static struct notifier_block cpufreq_notifier = { + .notifier_call = cpufreq_callback, +}; + +static int __init register_cpufreq_notifier(void) { - return rtc_mips_set_mmss(now.tv_sec); + return cpufreq_register_notifier(&cpufreq_notifier, + CPUFREQ_TRANSITION_NOTIFIER); } +core_initcall(register_cpufreq_notifier); + +#endif /* CONFIG_CPU_FREQ */ + +/* + * forward reference + */ +DEFINE_SPINLOCK(rtc_lock); +EXPORT_SYMBOL(rtc_lock); static int null_perf_irq(void) { @@ -70,20 +121,7 @@ EXPORT_SYMBOL(perf_irq); */ unsigned int mips_hpt_frequency; - -/* - * This function exists in order to cause an error due to a duplicate - * definition if platform code should have its own implementation. The hook - * to use instead is plat_time_init. plat_time_init does not receive the - * irqaction pointer argument anymore. This is because any function which - * initializes an interrupt timer now takes care of its own request_irq rsp. - * setup_irq calls and each clock_event_device should use its own - * struct irqrequest. - */ -void __init plat_timer_setup(void) -{ - BUG(); -} +EXPORT_SYMBOL_GPL(mips_hpt_frequency); static __init int cpu_has_mfc0_count_bug(void) { @@ -103,15 +141,10 @@ static __init int cpu_has_mfc0_count_bug(void) case CPU_R4400MC: /* * The published errata for the R4400 up to 3.0 say the CPU - * has the mfc0 from count bug. + * has the mfc0 from count bug. This seems the last version + * produced. */ - if ((current_cpu_data.processor_id & 0xff) <= 0x30) - return 1; - - /* - * we assume newer revisions are ok - */ - return 0; + return 1; } return 0; @@ -121,6 +154,14 @@ void __init time_init(void) { plat_time_init(); - if (!mips_clockevent_init() || !cpu_has_mfc0_count_bug()) + /* + * The use of the R4k timer as a clock event takes precedence; + * if reading the Count register might interfere with the timer + * interrupt, then we don't use the timer as a clock source. + * We may still use the timer as a clock source though if the + * timer interrupt isn't reliable; the interference doesn't + * matter then, because we don't use the interrupt. + */ + if (mips_clockevent_init() != 0 || !cpu_has_mfc0_count_bug()) init_mips_clocksource(); } diff --git a/arch/mips/kernel/topology.c b/arch/mips/kernel/topology.c index cf3eb61fad12..9429d85a4703 100644 --- a/arch/mips/kernel/topology.c +++ b/arch/mips/kernel/topology.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/cpu.h> #include <linux/cpumask.h> #include <linux/init.h> @@ -11,15 +12,10 @@ static int __init topology_init(void) { int i, ret; -#ifdef CONFIG_NUMA - for_each_online_node(i) - register_one_node(i); -#endif /* CONFIG_NUMA */ - for_each_present_cpu(i) { struct cpu *c = &per_cpu(cpu_devices, i); - c->hotpluggable = 1; + c->hotpluggable = !!i; ret = register_cpu(c, i); if (ret) printk(KERN_WARNING "topology_init: register_cpu %d " diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 0903d70b2cfe..8ec1e185b35c 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -10,20 +10,25 @@ * Kevin D. Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com * Copyright (C) 2002, 2003, 2004, 2005, 2007 Maciej W. Rozycki * Copyright (C) 2000, 2001, 2012 MIPS Technologies, Inc. All rights reserved. + * Copyright (C) 2014, Imagination Technologies Ltd. */ +#include <linux/bitops.h> #include <linux/bug.h> #include <linux/compiler.h> #include <linux/context_tracking.h> +#include <linux/cpu_pm.h> #include <linux/kexec.h> #include <linux/init.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/extable.h> #include <linux/mm.h> -#include <linux/sched.h> +#include <linux/sched/mm.h> +#include <linux/sched/debug.h> #include <linux/smp.h> #include <linux/spinlock.h> #include <linux/kallsyms.h> -#include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/interrupt.h> #include <linux/ptrace.h> #include <linux/kgdb.h> @@ -33,37 +38,47 @@ #include <linux/kdb.h> #include <linux/irq.h> #include <linux/perf_event.h> +#include <linux/string_choices.h> +#include <asm/addrspace.h> #include <asm/bootinfo.h> #include <asm/branch.h> #include <asm/break.h> #include <asm/cop2.h> #include <asm/cpu.h> +#include <asm/cpu-type.h> #include <asm/dsp.h> #include <asm/fpu.h> #include <asm/fpu_emulator.h> #include <asm/idle.h> +#include <asm/isa-rev.h> +#include <asm/mips-cps.h> +#include <asm/mips-r2-to-r6-emul.h> #include <asm/mipsregs.h> #include <asm/mipsmtregs.h> #include <asm/module.h> -#include <asm/pgtable.h> +#include <asm/msa.h> #include <asm/ptrace.h> +#include <asm/regdef.h> #include <asm/sections.h> +#include <asm/siginfo.h> #include <asm/tlbdebug.h> #include <asm/traps.h> -#include <asm/uaccess.h> +#include <linux/uaccess.h> #include <asm/watch.h> #include <asm/mmu_context.h> #include <asm/types.h> #include <asm/stacktrace.h> +#include <asm/tlbex.h> #include <asm/uasm.h> +#include <asm/mach-loongson64/cpucfg-emul.h> + +#include "access-helper.h" + extern void check_wait(void); -extern asmlinkage void rollback_handle_int(void); +extern asmlinkage void skipover_handle_int(void); extern asmlinkage void handle_int(void); -extern u32 handle_tlbl[]; -extern u32 handle_tlbs[]; -extern u32 handle_tlbm[]; extern asmlinkage void handle_adel(void); extern asmlinkage void handle_ades(void); extern asmlinkage void handle_ibe(void); @@ -71,47 +86,57 @@ extern asmlinkage void handle_dbe(void); extern asmlinkage void handle_sys(void); extern asmlinkage void handle_bp(void); extern asmlinkage void handle_ri(void); -extern asmlinkage void handle_ri_rdhwr_vivt(void); +extern asmlinkage void handle_ri_rdhwr_tlbp(void); extern asmlinkage void handle_ri_rdhwr(void); extern asmlinkage void handle_cpu(void); extern asmlinkage void handle_ov(void); extern asmlinkage void handle_tr(void); +extern asmlinkage void handle_msa_fpe(void); extern asmlinkage void handle_fpe(void); +extern asmlinkage void handle_ftlb(void); +extern asmlinkage void handle_gsexc(void); +extern asmlinkage void handle_msa(void); extern asmlinkage void handle_mdmx(void); extern asmlinkage void handle_watch(void); extern asmlinkage void handle_mt(void); extern asmlinkage void handle_dsp(void); extern asmlinkage void handle_mcheck(void); extern asmlinkage void handle_reserved(void); +extern void tlb_do_page_fault_0(void); void (*board_be_init)(void); -int (*board_be_handler)(struct pt_regs *regs, int is_fixup); +static int (*board_be_handler)(struct pt_regs *regs, int is_fixup); void (*board_nmi_handler_setup)(void); void (*board_ejtag_handler_setup)(void); void (*board_bind_eic_interrupt)(int irq, int regset); void (*board_ebase_setup)(void); -void __cpuinitdata(*board_cache_error_setup)(void); +void(*board_cache_error_setup)(void); + +void mips_set_be_handler(int (*handler)(struct pt_regs *regs, int is_fixup)) +{ + board_be_handler = handler; +} +EXPORT_SYMBOL_GPL(mips_set_be_handler); -static void show_raw_backtrace(unsigned long reg29) +static void show_raw_backtrace(unsigned long reg29, const char *loglvl, + bool user) { unsigned long *sp = (unsigned long *)(reg29 & ~3); unsigned long addr; - printk("Call Trace:"); + printk("%sCall Trace:", loglvl); #ifdef CONFIG_KALLSYMS - printk("\n"); + printk("%s\n", loglvl); #endif while (!kstack_end(sp)) { - unsigned long __user *p = - (unsigned long __user *)(unsigned long)sp++; - if (__get_user(addr, p)) { - printk(" (Bad stack address)"); + if (__get_addr(&addr, sp++, user)) { + printk("%s (Bad stack address)", loglvl); break; } if (__kernel_text_address(addr)) - print_ip_sym(addr); + print_ip_sym(loglvl, addr); } - printk("\n"); + printk("%s\n", loglvl); } #ifdef CONFIG_KALLSYMS @@ -124,7 +149,8 @@ static int __init set_raw_show_trace(char *str) __setup("raw_show_trace", set_raw_show_trace); #endif -static void show_backtrace(struct task_struct *task, const struct pt_regs *regs) +static void show_backtrace(struct task_struct *task, const struct pt_regs *regs, + const char *loglvl, bool user) { unsigned long sp = regs->regs[29]; unsigned long ra = regs->regs[31]; @@ -133,16 +159,16 @@ static void show_backtrace(struct task_struct *task, const struct pt_regs *regs) if (!task) task = current; - if (raw_show_trace || !__kernel_text_address(pc)) { - show_raw_backtrace(sp); + if (raw_show_trace || user_mode(regs) || !__kernel_text_address(pc)) { + show_raw_backtrace(sp, loglvl, user); return; } - printk("Call Trace:\n"); + printk("%sCall Trace:\n", loglvl); do { - print_ip_sym(pc); + print_ip_sym(loglvl, pc); pc = unwind_stack(task, &sp, pc, &ra); } while (pc); - printk("\n"); + pr_cont("\n"); } /* @@ -150,38 +176,42 @@ static void show_backtrace(struct task_struct *task, const struct pt_regs *regs) * with at least a bit of error checking ... */ static void show_stacktrace(struct task_struct *task, - const struct pt_regs *regs) + const struct pt_regs *regs, const char *loglvl, bool user) { const int field = 2 * sizeof(unsigned long); - long stackdata; + unsigned long stackdata; int i; - unsigned long __user *sp = (unsigned long __user *)regs->regs[29]; + unsigned long *sp = (unsigned long *)regs->regs[29]; - printk("Stack :"); + printk("%sStack :", loglvl); i = 0; while ((unsigned long) sp & (PAGE_SIZE - 1)) { - if (i && ((i % (64 / field)) == 0)) - printk("\n "); + if (i && ((i % (64 / field)) == 0)) { + pr_cont("\n"); + printk("%s ", loglvl); + } if (i > 39) { - printk(" ..."); + pr_cont(" ..."); break; } - if (__get_user(stackdata, sp++)) { - printk(" (Bad stack address)"); + if (__get_addr(&stackdata, sp++, user)) { + pr_cont(" (Bad stack address)"); break; } - printk(" %0*lx", field, stackdata); + pr_cont(" %0*lx", field, stackdata); i++; } - printk("\n"); - show_backtrace(task, regs); + pr_cont("\n"); + show_backtrace(task, regs, loglvl, user); } -void show_stack(struct task_struct *task, unsigned long *sp) +void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl) { struct pt_regs regs; + + regs.cp0_status = KSU_KERNEL; if (sp) { regs.regs[29] = (unsigned long)sp; regs.regs[31] = 0; @@ -191,41 +221,52 @@ void show_stack(struct task_struct *task, unsigned long *sp) regs.regs[29] = task->thread.reg29; regs.regs[31] = 0; regs.cp0_epc = task->thread.reg31; -#ifdef CONFIG_KGDB_KDB - } else if (atomic_read(&kgdb_active) != -1 && - kdb_current_regs) { - memcpy(®s, kdb_current_regs, sizeof(regs)); -#endif /* CONFIG_KGDB_KDB */ } else { prepare_frametrace(®s); } } - show_stacktrace(task, ®s); + show_stacktrace(task, ®s, loglvl, false); } -static void show_code(unsigned int __user *pc) +static void show_code(void *pc, bool user) { long i; - unsigned short __user *pc16 = NULL; + unsigned short *pc16 = NULL; - printk("\nCode:"); + printk("Code:"); if ((unsigned long)pc & 1) - pc16 = (unsigned short __user *)((unsigned long)pc & ~1); + pc16 = (u16 *)((unsigned long)pc & ~1); + for(i = -3 ; i < 6 ; i++) { - unsigned int insn; - if (pc16 ? __get_user(insn, pc16 + i) : __get_user(insn, pc + i)) { - printk(" (Bad address in epc)\n"); - break; + if (pc16) { + u16 insn16; + + if (__get_inst16(&insn16, pc16 + i, user)) + goto bad_address; + + pr_cont("%c%04x%c", (i?' ':'<'), insn16, (i?' ':'>')); + } else { + u32 insn32; + + if (__get_inst32(&insn32, (u32 *)pc + i, user)) + goto bad_address; + + pr_cont("%c%08x%c", (i?' ':'<'), insn32, (i?' ':'>')); } - printk("%c%0*x%c", (i?' ':'<'), pc16 ? 4 : 8, insn, (i?' ':'>')); } + pr_cont("\n"); + return; + +bad_address: + pr_cont(" (Bad address in epc)\n\n"); } static void __show_regs(const struct pt_regs *regs) { const int field = 2 * sizeof(unsigned long); unsigned int cause = regs->cp0_cause; + unsigned int exccode; int i; show_regs_print_info(KERN_DEFAULT); @@ -237,29 +278,30 @@ static void __show_regs(const struct pt_regs *regs) if ((i % 4) == 0) printk("$%2d :", i); if (i == 0) - printk(" %0*lx", field, 0UL); + pr_cont(" %0*lx", field, 0UL); else if (i == 26 || i == 27) - printk(" %*s", field, ""); + pr_cont(" %*s", field, ""); else - printk(" %0*lx", field, regs->regs[i]); + pr_cont(" %0*lx", field, regs->regs[i]); i++; if ((i % 4) == 0) - printk("\n"); + pr_cont("\n"); } #ifdef CONFIG_CPU_HAS_SMARTMIPS printk("Acx : %0*lx\n", field, regs->acx); #endif - printk("Hi : %0*lx\n", field, regs->hi); - printk("Lo : %0*lx\n", field, regs->lo); + if (MIPS_ISA_REV < 6) { + printk("Hi : %0*lx\n", field, regs->hi); + printk("Lo : %0*lx\n", field, regs->lo); + } /* * Saved cp0 registers */ printk("epc : %0*lx %pS\n", field, regs->cp0_epc, (void *) regs->cp0_epc); - printk(" %s\n", print_tainted()); printk("ra : %0*lx %pS\n", field, regs->regs[31], (void *) regs->regs[31]); @@ -267,51 +309,51 @@ static void __show_regs(const struct pt_regs *regs) if (cpu_has_3kex) { if (regs->cp0_status & ST0_KUO) - printk("KUo "); + pr_cont("KUo "); if (regs->cp0_status & ST0_IEO) - printk("IEo "); + pr_cont("IEo "); if (regs->cp0_status & ST0_KUP) - printk("KUp "); + pr_cont("KUp "); if (regs->cp0_status & ST0_IEP) - printk("IEp "); + pr_cont("IEp "); if (regs->cp0_status & ST0_KUC) - printk("KUc "); + pr_cont("KUc "); if (regs->cp0_status & ST0_IEC) - printk("IEc "); + pr_cont("IEc "); } else if (cpu_has_4kex) { if (regs->cp0_status & ST0_KX) - printk("KX "); + pr_cont("KX "); if (regs->cp0_status & ST0_SX) - printk("SX "); + pr_cont("SX "); if (regs->cp0_status & ST0_UX) - printk("UX "); + pr_cont("UX "); switch (regs->cp0_status & ST0_KSU) { case KSU_USER: - printk("USER "); + pr_cont("USER "); break; case KSU_SUPERVISOR: - printk("SUPERVISOR "); + pr_cont("SUPERVISOR "); break; case KSU_KERNEL: - printk("KERNEL "); + pr_cont("KERNEL "); break; default: - printk("BAD_MODE "); + pr_cont("BAD_MODE "); break; } if (regs->cp0_status & ST0_ERL) - printk("ERL "); + pr_cont("ERL "); if (regs->cp0_status & ST0_EXL) - printk("EXL "); + pr_cont("EXL "); if (regs->cp0_status & ST0_IE) - printk("IE "); + pr_cont("IE "); } - printk("\n"); + pr_cont("\n"); - printk("Cause : %08x\n", cause); + exccode = (cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE; + printk("Cause : %08x (ExcCode %02x)\n", cause, exccode); - cause = (cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE; - if (1 <= cause && cause <= 5) + if (1 <= exccode && exccode <= 5) printk("BadVA : %0*lx\n", field, regs->cp0_badvaddr); printk("PrId : %08x (%s)\n", read_c0_prid(), @@ -323,7 +365,8 @@ static void __show_regs(const struct pt_regs *regs) */ void show_regs(struct pt_regs *regs) { - __show_regs((struct pt_regs *)regs); + __show_regs(regs); + dump_stack(); } void show_registers(struct pt_regs *regs) @@ -343,40 +386,27 @@ void show_registers(struct pt_regs *regs) printk("*HwTLS: %0*lx\n", field, tls); } - show_stacktrace(current, regs); - show_code((unsigned int __user *) regs->cp0_epc); + show_stacktrace(current, regs, KERN_DEFAULT, user_mode(regs)); + show_code((void *)regs->cp0_epc, user_mode(regs)); printk("\n"); } -static int regs_to_trapnr(struct pt_regs *regs) -{ - return (regs->cp0_cause >> 2) & 0x1f; -} - static DEFINE_RAW_SPINLOCK(die_lock); void __noreturn die(const char *str, struct pt_regs *regs) { static int die_counter; int sig = SIGSEGV; -#ifdef CONFIG_MIPS_MT_SMTC - unsigned long dvpret; -#endif /* CONFIG_MIPS_MT_SMTC */ oops_enter(); - if (notify_die(DIE_OOPS, str, regs, 0, regs_to_trapnr(regs), SIGSEGV) == NOTIFY_STOP) + if (notify_die(DIE_OOPS, str, regs, 0, current->thread.trap_nr, + SIGSEGV) == NOTIFY_STOP) sig = 0; console_verbose(); raw_spin_lock_irq(&die_lock); -#ifdef CONFIG_MIPS_MT_SMTC - dvpret = dvpe(); -#endif /* CONFIG_MIPS_MT_SMTC */ bust_spinlocks(1); -#ifdef CONFIG_MIPS_MT_SMTC - mips_mt_regdump(dvpret); -#endif /* CONFIG_MIPS_MT_SMTC */ printk("%s[#%d]:\n", str, ++die_counter); show_registers(regs); @@ -388,16 +418,13 @@ void __noreturn die(const char *str, struct pt_regs *regs) if (in_interrupt()) panic("Fatal exception in interrupt"); - if (panic_on_oops) { - printk(KERN_EMERG "Fatal exception: panic in 5 seconds"); - ssleep(5); + if (panic_on_oops) panic("Fatal exception"); - } if (regs && kexec_should_crash(current)) crash_kexec(regs); - do_exit(sig); + make_task_dead(sig); } extern struct exception_table_entry __start___dbe_table[]; @@ -412,7 +439,8 @@ static const struct exception_table_entry *search_dbe_tables(unsigned long addr) { const struct exception_table_entry *e; - e = search_extable(__start___dbe_table, __stop___dbe_table - 1, addr); + e = search_extable(__start___dbe_table, + __stop___dbe_table - __start___dbe_table, addr); if (!e) e = search_module_dbetables(addr); return e; @@ -436,6 +464,8 @@ asmlinkage void do_be(struct pt_regs *regs) if (board_be_handler) action = board_be_handler(regs, fixup != NULL); + else + mips_cm_error_report(); switch (action) { case MIPS_BE_DISCARD: @@ -456,12 +486,12 @@ asmlinkage void do_be(struct pt_regs *regs) printk(KERN_ALERT "%s bus error, epc == %0*lx, ra == %0*lx\n", data ? "Data" : "Instruction", field, regs->cp0_epc, field, regs->regs[31]); - if (notify_die(DIE_OOPS, "bus error", regs, 0, regs_to_trapnr(regs), SIGBUS) - == NOTIFY_STOP) + if (notify_die(DIE_OOPS, "bus error", regs, 0, current->thread.trap_nr, + SIGBUS) == NOTIFY_STOP) goto out; die_if_kernel("Oops", regs); - force_sig(SIGBUS, current); + force_sig(SIGBUS); out: exception_exit(prev_state); @@ -611,18 +641,18 @@ static int simulate_rdhwr(struct pt_regs *regs, int rd, int rt) perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); switch (rd) { - case 0: /* CPU number */ + case MIPS_HWR_CPUNUM: /* CPU number */ regs->regs[rt] = smp_processor_id(); return 0; - case 1: /* SYNCI length */ + case MIPS_HWR_SYNCISTEP: /* SYNCI length */ regs->regs[rt] = min(current_cpu_data.dcache.linesz, current_cpu_data.icache.linesz); return 0; - case 2: /* Read count register */ + case MIPS_HWR_CC: /* Read count register */ regs->regs[rt] = read_c0_count(); return 0; - case 3: /* Count register resolution */ - switch (current_cpu_data.cputype) { + case MIPS_HWR_CCRES: /* Count register resolution */ + switch (current_cpu_type()) { case CPU_20KC: case CPU_25KF: regs->regs[rt] = 1; @@ -631,7 +661,7 @@ static int simulate_rdhwr(struct pt_regs *regs, int rd, int rt) regs->regs[rt] = 2; } return 0; - case 29: + case MIPS_HWR_ULR: /* Read UserLocal register */ regs->regs[rt] = ti->tp_value; return 0; default: @@ -653,7 +683,7 @@ static int simulate_rdhwr_normal(struct pt_regs *regs, unsigned int opcode) return -1; } -static int simulate_rdhwr_mm(struct pt_regs *regs, unsigned short opcode) +static int simulate_rdhwr_mm(struct pt_regs *regs, unsigned int opcode) { if ((opcode & MM_POOL32A_FUNC) == MM_RDHWR) { int rd = (opcode & MM_RS) >> 16; @@ -677,64 +707,190 @@ static int simulate_sync(struct pt_regs *regs, unsigned int opcode) return -1; /* Must be something else ... */ } +/* + * Loongson-3 CSR instructions emulation + */ + +#ifdef CONFIG_CPU_LOONGSON3_CPUCFG_EMULATION + +#define LWC2 0xc8000000 +#define RS BASE +#define CSR_OPCODE2 0x00000118 +#define CSR_OPCODE2_MASK 0x000007ff +#define CSR_FUNC_MASK RT +#define CSR_FUNC_CPUCFG 0x8 + +static int simulate_loongson3_cpucfg(struct pt_regs *regs, + unsigned int opcode) +{ + int op = opcode & OPCODE; + int op2 = opcode & CSR_OPCODE2_MASK; + int csr_func = (opcode & CSR_FUNC_MASK) >> 16; + + if (op == LWC2 && op2 == CSR_OPCODE2 && csr_func == CSR_FUNC_CPUCFG) { + int rd = (opcode & RD) >> 11; + int rs = (opcode & RS) >> 21; + __u64 sel = regs->regs[rs]; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); + + /* Do not emulate on unsupported core models. */ + preempt_disable(); + if (!loongson3_cpucfg_emulation_enabled(¤t_cpu_data)) { + preempt_enable(); + return -1; + } + regs->regs[rd] = loongson3_cpucfg_read_synthesized( + ¤t_cpu_data, sel); + preempt_enable(); + return 0; + } + + /* Not ours. */ + return -1; +} +#endif /* CONFIG_CPU_LOONGSON3_CPUCFG_EMULATION */ + asmlinkage void do_ov(struct pt_regs *regs) { enum ctx_state prev_state; - siginfo_t info; prev_state = exception_enter(); die_if_kernel("Integer overflow", regs); - info.si_code = FPE_INTOVF; - info.si_signo = SIGFPE; - info.si_errno = 0; - info.si_addr = (void __user *) regs->cp0_epc; - force_sig_info(SIGFPE, &info, current); + force_sig_fault(SIGFPE, FPE_INTOVF, (void __user *)regs->cp0_epc); exception_exit(prev_state); } -int process_fpemu_return(int sig, void __user *fault_addr) +#ifdef CONFIG_MIPS_FP_SUPPORT + +/* + * Send SIGFPE according to FCSR Cause bits, which must have already + * been masked against Enable bits. This is impotant as Inexact can + * happen together with Overflow or Underflow, and `ptrace' can set + * any bits. + */ +void force_fcr31_sig(unsigned long fcr31, void __user *fault_addr, + struct task_struct *tsk) { - if (sig == SIGSEGV || sig == SIGBUS) { - struct siginfo si = {0}; - si.si_addr = fault_addr; - si.si_signo = sig; - if (sig == SIGSEGV) { - if (find_vma(current->mm, (unsigned long)fault_addr)) - si.si_code = SEGV_ACCERR; - else - si.si_code = SEGV_MAPERR; - } else { - si.si_code = BUS_ADRERR; - } - force_sig_info(sig, &si, current); + int si_code = FPE_FLTUNK; + + if (fcr31 & FPU_CSR_INV_X) + si_code = FPE_FLTINV; + else if (fcr31 & FPU_CSR_DIV_X) + si_code = FPE_FLTDIV; + else if (fcr31 & FPU_CSR_OVF_X) + si_code = FPE_FLTOVF; + else if (fcr31 & FPU_CSR_UDF_X) + si_code = FPE_FLTUND; + else if (fcr31 & FPU_CSR_INE_X) + si_code = FPE_FLTRES; + + force_sig_fault_to_task(SIGFPE, si_code, fault_addr, tsk); +} + +int process_fpemu_return(int sig, void __user *fault_addr, unsigned long fcr31) +{ + int si_code; + + switch (sig) { + case 0: + return 0; + + case SIGFPE: + force_fcr31_sig(fcr31, fault_addr, current); return 1; - } else if (sig) { - force_sig(sig, current); + + case SIGBUS: + force_sig_fault(SIGBUS, BUS_ADRERR, fault_addr); + return 1; + + case SIGSEGV: + mmap_read_lock(current->mm); + if (vma_lookup(current->mm, (unsigned long)fault_addr)) + si_code = SEGV_ACCERR; + else + si_code = SEGV_MAPERR; + mmap_read_unlock(current->mm); + force_sig_fault(SIGSEGV, si_code, fault_addr); + return 1; + + default: + force_sig(sig); return 1; - } else { - return 0; } } +static int simulate_fp(struct pt_regs *regs, unsigned int opcode, + unsigned long old_epc, unsigned long old_ra) +{ + union mips_instruction inst = { .word = opcode }; + void __user *fault_addr; + unsigned long fcr31; + int sig; + + /* If it's obviously not an FP instruction, skip it */ + switch (inst.i_format.opcode) { + case cop1_op: + case cop1x_op: + case lwc1_op: + case ldc1_op: + case swc1_op: + case sdc1_op: + break; + + default: + return -1; + } + + /* + * do_ri skipped over the instruction via compute_return_epc, undo + * that for the FPU emulator. + */ + regs->cp0_epc = old_epc; + regs->regs[31] = old_ra; + + /* Run the emulator */ + sig = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, + &fault_addr); + + /* + * We can't allow the emulated instruction to leave any + * enabled Cause bits set in $fcr31. + */ + fcr31 = mask_fcr31_x(current->thread.fpu.fcr31); + current->thread.fpu.fcr31 &= ~fcr31; + + /* Restore the hardware register state */ + own_fpu(1); + + /* Send a signal if required. */ + process_fpemu_return(sig, fault_addr, fcr31); + + return 0; +} + /* * XXX Delayed fp exceptions when doing a lazy ctx switch XXX */ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31) { enum ctx_state prev_state; - siginfo_t info = {0}; + void __user *fault_addr; + int sig; prev_state = exception_enter(); - if (notify_die(DIE_FP, "FP exception", regs, 0, regs_to_trapnr(regs), SIGFPE) - == NOTIFY_STOP) + if (notify_die(DIE_FP, "FP exception", regs, 0, current->thread.trap_nr, + SIGFPE) == NOTIFY_STOP) goto out; + + /* Clear FCSR.Cause before enabling interrupts */ + write_32bit_cp1_register(CP1_STATUS, fcr31 & ~mask_fcr31_x(fcr31)); + local_irq_enable(); + die_if_kernel("FP exception in kernel code", regs); if (fcr31 & FPU_CSR_UNI_X) { - int sig; - void __user *fault_addr = NULL; - /* * Unimplemented operation exception. If we've got the full * software emulator on-board, let's use it... @@ -745,59 +901,84 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31) * register operands before invoking the emulator, which seems * a bit extreme for what should be an infrequent event. */ - /* Ensure 'resume' not overwrite saved fp context again. */ - lose_fpu(1); /* Run the emulator */ sig = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, &fault_addr); /* - * We can't allow the emulated instruction to leave any of - * the cause bit set in $fcr31. + * We can't allow the emulated instruction to leave any + * enabled Cause bits set in $fcr31. */ - current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X; + fcr31 = mask_fcr31_x(current->thread.fpu.fcr31); + current->thread.fpu.fcr31 &= ~fcr31; /* Restore the hardware register state */ own_fpu(1); /* Using the FPU again. */ + } else { + sig = SIGFPE; + fault_addr = (void __user *) regs->cp0_epc; + } - /* If something went wrong, signal */ - process_fpemu_return(sig, fault_addr); - - goto out; - } else if (fcr31 & FPU_CSR_INV_X) - info.si_code = FPE_FLTINV; - else if (fcr31 & FPU_CSR_DIV_X) - info.si_code = FPE_FLTDIV; - else if (fcr31 & FPU_CSR_OVF_X) - info.si_code = FPE_FLTOVF; - else if (fcr31 & FPU_CSR_UDF_X) - info.si_code = FPE_FLTUND; - else if (fcr31 & FPU_CSR_INE_X) - info.si_code = FPE_FLTRES; - else - info.si_code = __SI_FAULT; - info.si_signo = SIGFPE; - info.si_errno = 0; - info.si_addr = (void __user *) regs->cp0_epc; - force_sig_info(SIGFPE, &info, current); + /* Send a signal if required. */ + process_fpemu_return(sig, fault_addr, fcr31); out: exception_exit(prev_state); } -static void do_trap_or_bp(struct pt_regs *regs, unsigned int code, +/* + * MIPS MT processors may have fewer FPU contexts than CPU threads. If we've + * emulated more than some threshold number of instructions, force migration to + * a "CPU" that has FP support. + */ +static void mt_ase_fp_affinity(void) +{ +#ifdef CONFIG_MIPS_MT_FPAFF + if (mt_fpemul_threshold > 0 && + ((current->thread.emulated_fp++ > mt_fpemul_threshold))) { + /* + * If there's no FPU present, or if the application has already + * restricted the allowed set to exclude any CPUs with FPUs, + * we'll skip the procedure. + */ + if (cpumask_intersects(¤t->cpus_mask, &mt_fpu_cpumask)) { + cpumask_t tmask; + + current->thread.user_cpus_allowed + = current->cpus_mask; + cpumask_and(&tmask, ¤t->cpus_mask, + &mt_fpu_cpumask); + set_cpus_allowed_ptr(current, &tmask); + set_thread_flag(TIF_FPUBOUND); + } + } +#endif /* CONFIG_MIPS_MT_FPAFF */ +} + +#else /* !CONFIG_MIPS_FP_SUPPORT */ + +static int simulate_fp(struct pt_regs *regs, unsigned int opcode, + unsigned long old_epc, unsigned long old_ra) +{ + return -1; +} + +#endif /* !CONFIG_MIPS_FP_SUPPORT */ + +void do_trap_or_bp(struct pt_regs *regs, unsigned int code, int si_code, const char *str) { - siginfo_t info; char b[40]; #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP - if (kgdb_ll_trap(DIE_TRAP, str, regs, code, regs_to_trapnr(regs), SIGTRAP) == NOTIFY_STOP) + if (kgdb_ll_trap(DIE_TRAP, str, regs, code, current->thread.trap_nr, + SIGTRAP) == NOTIFY_STOP) return; #endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */ - if (notify_die(DIE_TRAP, str, regs, code, regs_to_trapnr(regs), SIGTRAP) == NOTIFY_STOP) + if (notify_die(DIE_TRAP, str, regs, code, current->thread.trap_nr, + SIGTRAP) == NOTIFY_STOP) return; /* @@ -811,24 +992,19 @@ static void do_trap_or_bp(struct pt_regs *regs, unsigned int code, case BRK_DIVZERO: scnprintf(b, sizeof(b), "%s instruction in kernel code", str); die_if_kernel(b, regs); - if (code == BRK_DIVZERO) - info.si_code = FPE_INTDIV; - else - info.si_code = FPE_INTOVF; - info.si_signo = SIGFPE; - info.si_errno = 0; - info.si_addr = (void __user *) regs->cp0_epc; - force_sig_info(SIGFPE, &info, current); + force_sig_fault(SIGFPE, + code == BRK_DIVZERO ? FPE_INTDIV : FPE_INTOVF, + (void __user *) regs->cp0_epc); break; case BRK_BUG: die_if_kernel("Kernel bug detected", regs); - force_sig(SIGTRAP, current); + force_sig(SIGTRAP); break; case BRK_MEMU: /* - * Address errors may be deliberately induced by the FPU - * emulator to retake control of the CPU after executing the - * instruction in the delay slot of an emulated branch. + * This breakpoint code is used by the FPU emulator to retake + * control of the CPU after executing the instruction from the + * delay slot of an emulated branch. * * Terminate if exception was recognized as a delay slot return * otherwise handle as normal. @@ -837,42 +1013,51 @@ static void do_trap_or_bp(struct pt_regs *regs, unsigned int code, return; die_if_kernel("Math emu break/trap", regs); - force_sig(SIGTRAP, current); + force_sig(SIGTRAP); break; default: scnprintf(b, sizeof(b), "%s instruction in kernel code", str); die_if_kernel(b, regs); - force_sig(SIGTRAP, current); + if (si_code) { + force_sig_fault(SIGTRAP, si_code, NULL); + } else { + force_sig(SIGTRAP); + } } } asmlinkage void do_bp(struct pt_regs *regs) { + unsigned long epc = msk_isa16_mode(exception_epc(regs)); unsigned int opcode, bcode; enum ctx_state prev_state; - unsigned long epc; - u16 instr[2]; + bool user = user_mode(regs); prev_state = exception_enter(); + current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f; if (get_isa16_mode(regs->cp0_epc)) { - /* Calculate EPC. */ - epc = exception_epc(regs); - if (cpu_has_mmips) { - if ((__get_user(instr[0], (u16 __user *)msk_isa16_mode(epc)) || - (__get_user(instr[1], (u16 __user *)msk_isa16_mode(epc + 2))))) - goto out_sigsegv; - opcode = (instr[0] << 16) | instr[1]; + u16 instr[2]; + + if (__get_inst16(&instr[0], (u16 *)epc, user)) + goto out_sigsegv; + + if (!cpu_has_mmips) { + /* MIPS16e mode */ + bcode = (instr[0] >> 5) & 0x3f; + } else if (mm_insn_16bit(instr[0])) { + /* 16-bit microMIPS BREAK */ + bcode = instr[0] & 0xf; } else { - /* MIPS16e mode */ - if (__get_user(instr[0], (u16 __user *)msk_isa16_mode(epc))) + /* 32-bit microMIPS BREAK */ + if (__get_inst16(&instr[1], (u16 *)(epc + 2), user)) goto out_sigsegv; - bcode = (instr[0] >> 6) & 0x3f; - do_trap_or_bp(regs, bcode, "Break"); - goto out; + opcode = (instr[0] << 16) | instr[1]; + bcode = (opcode >> 6) & ((1 << 20) - 1); } } else { - if (__get_user(opcode, (unsigned int __user *) exception_epc(regs))) + if (__get_inst32(&opcode, (u32 *)epc, user)) goto out_sigsegv; + bcode = (opcode >> 6) & ((1 << 20) - 1); } /* @@ -881,22 +1066,35 @@ asmlinkage void do_bp(struct pt_regs *regs) * Gas is bug-compatible, but not always, grrr... * We handle both cases with a simple heuristics. --macro */ - bcode = ((opcode >> 6) & ((1 << 20) - 1)); if (bcode >= (1 << 10)) - bcode >>= 10; + bcode = ((bcode & ((1 << 10) - 1)) << 10) | (bcode >> 10); /* * notify the kprobe handlers, if instruction is likely to * pertain to them. */ switch (bcode) { + case BRK_UPROBE: + if (notify_die(DIE_UPROBE, "uprobe", regs, bcode, + current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) + goto out; + else + break; + case BRK_UPROBE_XOL: + if (notify_die(DIE_UPROBE_XOL, "uprobe_xol", regs, bcode, + current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) + goto out; + else + break; case BRK_KPROBE_BP: - if (notify_die(DIE_BREAK, "debug", regs, bcode, regs_to_trapnr(regs), SIGTRAP) == NOTIFY_STOP) + if (notify_die(DIE_BREAK, "debug", regs, bcode, + current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) goto out; else break; case BRK_KPROBE_SSTEPBP: - if (notify_die(DIE_SSTEPBP, "single_step", regs, bcode, regs_to_trapnr(regs), SIGTRAP) == NOTIFY_STOP) + if (notify_die(DIE_SSTEPBP, "single_step", regs, bcode, + current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) goto out; else break; @@ -904,14 +1102,14 @@ asmlinkage void do_bp(struct pt_regs *regs) break; } - do_trap_or_bp(regs, bcode, "Break"); + do_trap_or_bp(regs, bcode, TRAP_BRKPT, "Break"); out: exception_exit(prev_state); return; out_sigsegv: - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); goto out; } @@ -920,33 +1118,35 @@ asmlinkage void do_tr(struct pt_regs *regs) u32 opcode, tcode = 0; enum ctx_state prev_state; u16 instr[2]; + bool user = user_mode(regs); unsigned long epc = msk_isa16_mode(exception_epc(regs)); prev_state = exception_enter(); + current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f; if (get_isa16_mode(regs->cp0_epc)) { - if (__get_user(instr[0], (u16 __user *)(epc + 0)) || - __get_user(instr[1], (u16 __user *)(epc + 2))) + if (__get_inst16(&instr[0], (u16 *)(epc + 0), user) || + __get_inst16(&instr[1], (u16 *)(epc + 2), user)) goto out_sigsegv; opcode = (instr[0] << 16) | instr[1]; /* Immediate versions don't provide a code. */ if (!(opcode & OPCODE)) tcode = (opcode >> 12) & ((1 << 4) - 1); } else { - if (__get_user(opcode, (u32 __user *)epc)) + if (__get_inst32(&opcode, (u32 *)epc, user)) goto out_sigsegv; /* Immediate versions don't provide a code. */ if (!(opcode & OPCODE)) tcode = (opcode >> 6) & ((1 << 10) - 1); } - do_trap_or_bp(regs, tcode, "Trap"); + do_trap_or_bp(regs, tcode, 0, "Trap"); out: exception_exit(prev_state); return; out_sigsegv: - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); goto out; } @@ -959,9 +1159,37 @@ asmlinkage void do_ri(struct pt_regs *regs) unsigned int opcode = 0; int status = -1; + /* + * Avoid any kernel code. Just emulate the R2 instruction + * as quickly as possible. + */ + if (mipsr2_emulation && cpu_has_mips_r6 && + likely(user_mode(regs)) && + likely(get_user(opcode, epc) >= 0)) { + unsigned long fcr31 = 0; + + status = mipsr2_decoder(regs, opcode, &fcr31); + switch (status) { + case 0: + case SIGEMT: + return; + case SIGILL: + goto no_r2_instr; + default: + process_fpemu_return(status, + ¤t->thread.cp0_baduaddr, + fcr31); + return; + } + } + +no_r2_instr: + prev_state = exception_enter(); - if (notify_die(DIE_RI, "RI Fault", regs, 0, regs_to_trapnr(regs), SIGILL) - == NOTIFY_STOP) + current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f; + + if (notify_die(DIE_RI, "RI Fault", regs, 0, current->thread.trap_nr, + SIGILL) == NOTIFY_STOP) goto out; die_if_kernel("Reserved instruction in kernel code", regs); @@ -969,18 +1197,7 @@ asmlinkage void do_ri(struct pt_regs *regs) if (unlikely(compute_return_epc(regs) < 0)) goto out; - if (get_isa16_mode(regs->cp0_epc)) { - unsigned short mmop[2] = { 0 }; - - if (unlikely(get_user(mmop[0], epc) < 0)) - status = SIGSEGV; - if (unlikely(get_user(mmop[1], epc) < 0)) - status = SIGSEGV; - opcode = (mmop[0] << 16) | mmop[1]; - - if (status < 0) - status = simulate_rdhwr_mm(regs, opcode); - } else { + if (!get_isa16_mode(regs->cp0_epc)) { if (unlikely(get_user(opcode, epc) < 0)) status = SIGSEGV; @@ -992,6 +1209,26 @@ asmlinkage void do_ri(struct pt_regs *regs) if (status < 0) status = simulate_sync(regs, opcode); + + if (status < 0) + status = simulate_fp(regs, opcode, old_epc, old31); + +#ifdef CONFIG_CPU_LOONGSON3_CPUCFG_EMULATION + if (status < 0) + status = simulate_loongson3_cpucfg(regs, opcode); +#endif + } else if (cpu_has_mmips) { + unsigned short mmop[2] = { 0 }; + + if (unlikely(get_user(mmop[0], (u16 __user *)epc + 0) < 0)) + status = SIGSEGV; + if (unlikely(get_user(mmop[1], (u16 __user *)epc + 1) < 0)) + status = SIGSEGV; + opcode = mmop[0]; + opcode = (opcode << 16) | mmop[1]; + + if (status < 0) + status = simulate_rdhwr_mm(regs, opcode); } if (status < 0) @@ -1000,7 +1237,7 @@ asmlinkage void do_ri(struct pt_regs *regs) if (unlikely(status > 0)) { regs->cp0_epc = old_epc; /* Undo skip-over. */ regs->regs[31] = old31; - force_sig(status, current); + force_sig(status); } out: @@ -1008,35 +1245,6 @@ out: } /* - * MIPS MT processors may have fewer FPU contexts than CPU threads. If we've - * emulated more than some threshold number of instructions, force migration to - * a "CPU" that has FP support. - */ -static void mt_ase_fp_affinity(void) -{ -#ifdef CONFIG_MIPS_MT_FPAFF - if (mt_fpemul_threshold > 0 && - ((current->thread.emulated_fp++ > mt_fpemul_threshold))) { - /* - * If there's no FPU present, or if the application has already - * restricted the allowed set to exclude any CPUs with FPUs, - * we'll skip the procedure. - */ - if (cpus_intersects(current->cpus_allowed, mt_fpu_cpumask)) { - cpumask_t tmask; - - current->thread.user_cpus_allowed - = current->cpus_allowed; - cpus_and(tmask, current->cpus_allowed, - mt_fpu_cpumask); - set_cpus_allowed_ptr(current, &tmask); - set_thread_flag(TIF_FPUBOUND); - } - } -#endif /* CONFIG_MIPS_MT_FPAFF */ -} - -/* * No lock; only written during early bootup by CPU 0. */ static RAW_NOTIFIER_HEAD(cu2_chain); @@ -1058,11 +1266,141 @@ static int default_cu2_call(struct notifier_block *nfb, unsigned long action, die_if_kernel("COP2: Unhandled kernel unaligned access or invalid " "instruction", regs); - force_sig(SIGILL, current); + force_sig(SIGILL); return NOTIFY_OK; } +#ifdef CONFIG_MIPS_FP_SUPPORT + +static int enable_restore_fp_context(int msa) +{ + int err, was_fpu_owner, prior_msa; + bool first_fp; + + /* Initialize context if it hasn't been used already */ + first_fp = init_fp_ctx(current); + + if (first_fp) { + preempt_disable(); + err = own_fpu_inatomic(1); + if (msa && !err) { + enable_msa(); + /* + * with MSA enabled, userspace can see MSACSR + * and MSA regs, but the values in them are from + * other task before current task, restore them + * from saved fp/msa context + */ + write_msa_csr(current->thread.fpu.msacsr); + /* + * own_fpu_inatomic(1) just restore low 64bit, + * fix the high 64bit + */ + init_msa_upper(); + set_thread_flag(TIF_USEDMSA); + set_thread_flag(TIF_MSA_CTX_LIVE); + } + preempt_enable(); + return err; + } + + /* + * This task has formerly used the FP context. + * + * If this thread has no live MSA vector context then we can simply + * restore the scalar FP context. If it has live MSA vector context + * (that is, it has or may have used MSA since last performing a + * function call) then we'll need to restore the vector context. This + * applies even if we're currently only executing a scalar FP + * instruction. This is because if we were to later execute an MSA + * instruction then we'd either have to: + * + * - Restore the vector context & clobber any registers modified by + * scalar FP instructions between now & then. + * + * or + * + * - Not restore the vector context & lose the most significant bits + * of all vector registers. + * + * Neither of those options is acceptable. We cannot restore the least + * significant bits of the registers now & only restore the most + * significant bits later because the most significant bits of any + * vector registers whose aliased FP register is modified now will have + * been zeroed. We'd have no way to know that when restoring the vector + * context & thus may load an outdated value for the most significant + * bits of a vector register. + */ + if (!msa && !thread_msa_context_live()) + return own_fpu(1); + + /* + * This task is using or has previously used MSA. Thus we require + * that Status.FR == 1. + */ + preempt_disable(); + was_fpu_owner = is_fpu_owner(); + err = own_fpu_inatomic(0); + if (err) + goto out; + + enable_msa(); + write_msa_csr(current->thread.fpu.msacsr); + set_thread_flag(TIF_USEDMSA); + + /* + * If this is the first time that the task is using MSA and it has + * previously used scalar FP in this time slice then we already nave + * FP context which we shouldn't clobber. We do however need to clear + * the upper 64b of each vector register so that this task has no + * opportunity to see data left behind by another. + */ + prior_msa = test_and_set_thread_flag(TIF_MSA_CTX_LIVE); + if (!prior_msa && was_fpu_owner) { + init_msa_upper(); + + goto out; + } + + if (!prior_msa) { + /* + * Restore the least significant 64b of each vector register + * from the existing scalar FP context. + */ + _restore_fp(current); + + /* + * The task has not formerly used MSA, so clear the upper 64b + * of each vector register such that it cannot see data left + * behind by another task. + */ + init_msa_upper(); + } else { + /* We need to restore the vector context. */ + restore_msa(current); + + /* Restore the scalar FP control & status register */ + if (!was_fpu_owner) + write_32bit_cp1_register(CP1_STATUS, + current->thread.fpu.fcr31); + } + +out: + preempt_enable(); + + return 0; +} + +#else /* !CONFIG_MIPS_FP_SUPPORT */ + +static int enable_restore_fp_context(int msa) +{ + return SIGILL; +} + +#endif /* CONFIG_MIPS_FP_SUPPORT */ + asmlinkage void do_cpu(struct pt_regs *regs) { enum ctx_state prev_state; @@ -1071,7 +1409,6 @@ asmlinkage void do_cpu(struct pt_regs *regs) unsigned int opcode; unsigned int cpid; int status; - unsigned long __maybe_unused flags; prev_state = exception_enter(); cpid = (regs->cp0_cause >> CAUSEB_CE) & 3; @@ -1088,28 +1425,14 @@ asmlinkage void do_cpu(struct pt_regs *regs) status = -1; if (unlikely(compute_return_epc(regs) < 0)) - goto out; - - if (get_isa16_mode(regs->cp0_epc)) { - unsigned short mmop[2] = { 0 }; - - if (unlikely(get_user(mmop[0], epc) < 0)) - status = SIGSEGV; - if (unlikely(get_user(mmop[1], epc) < 0)) - status = SIGSEGV; - opcode = (mmop[0] << 16) | mmop[1]; + break; - if (status < 0) - status = simulate_rdhwr_mm(regs, opcode); - } else { + if (!get_isa16_mode(regs->cp0_epc)) { if (unlikely(get_user(opcode, epc) < 0)) status = SIGSEGV; if (!cpu_has_llsc && status < 0) status = simulate_llsc(regs, opcode); - - if (status < 0) - status = simulate_rdhwr_normal(regs, opcode); } if (status < 0) @@ -1118,56 +1441,108 @@ asmlinkage void do_cpu(struct pt_regs *regs) if (unlikely(status > 0)) { regs->cp0_epc = old_epc; /* Undo skip-over. */ regs->regs[31] = old31; - force_sig(status, current); + force_sig(status); } - goto out; + break; +#ifdef CONFIG_MIPS_FP_SUPPORT case 3: /* - * Old (MIPS I and MIPS II) processors will set this code - * for COP1X opcode instructions that replaced the original - * COP3 space. We don't limit COP1 space instructions in - * the emulator according to the CPU ISA, so we want to - * treat COP1X instructions consistently regardless of which - * code the CPU chose. Therefore we redirect this trap to - * the FP emulator too. - * - * Then some newer FPU-less processors use this code - * erroneously too, so they are covered by this choice - * as well. + * The COP3 opcode space and consequently the CP0.Status.CU3 + * bit and the CP0.Cause.CE=3 encoding have been removed as + * of the MIPS III ISA. From the MIPS IV and MIPS32r2 ISAs + * up the space has been reused for COP1X instructions, that + * are enabled by the CP0.Status.CU1 bit and consequently + * use the CP0.Cause.CE=1 encoding for Coprocessor Unusable + * exceptions. Some FPU-less processors that implement one + * of these ISAs however use this code erroneously for COP1X + * instructions. Therefore we redirect this trap to the FP + * emulator too. */ - if (raw_cpu_has_fpu) + if (raw_cpu_has_fpu || !cpu_has_mips_4_5_64_r2_r6) { + force_sig(SIGILL); break; - /* Fall through. */ - - case 1: - if (used_math()) /* Using the FPU again. */ - own_fpu(1); - else { /* First time FPU user. */ - init_fpu(); - set_used_math(); } + fallthrough; + case 1: { + void __user *fault_addr; + unsigned long fcr31; + int err, sig; - if (!raw_cpu_has_fpu) { - int sig; - void __user *fault_addr = NULL; - sig = fpu_emulator_cop1Handler(regs, - ¤t->thread.fpu, - 0, &fault_addr); - if (!process_fpemu_return(sig, fault_addr)) - mt_ase_fp_affinity(); - } + err = enable_restore_fp_context(0); - goto out; + if (raw_cpu_has_fpu && !err) + break; + + sig = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 0, + &fault_addr); + + /* + * We can't allow the emulated instruction to leave + * any enabled Cause bits set in $fcr31. + */ + fcr31 = mask_fcr31_x(current->thread.fpu.fcr31); + current->thread.fpu.fcr31 &= ~fcr31; + + /* Send a signal if required. */ + if (!process_fpemu_return(sig, fault_addr, fcr31) && !err) + mt_ase_fp_affinity(); + + break; + } +#else /* CONFIG_MIPS_FP_SUPPORT */ + case 1: + case 3: + force_sig(SIGILL); + break; +#endif /* CONFIG_MIPS_FP_SUPPORT */ case 2: raw_notifier_call_chain(&cu2_chain, CU2_EXCEPTION, regs); + break; + } + + exception_exit(prev_state); +} + +asmlinkage void do_msa_fpe(struct pt_regs *regs, unsigned int msacsr) +{ + enum ctx_state prev_state; + + prev_state = exception_enter(); + current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f; + if (notify_die(DIE_MSAFP, "MSA FP exception", regs, 0, + current->thread.trap_nr, SIGFPE) == NOTIFY_STOP) + goto out; + + /* Clear MSACSR.Cause before enabling interrupts */ + write_msa_csr(msacsr & ~MSA_CSR_CAUSEF); + local_irq_enable(); + + die_if_kernel("do_msa_fpe invoked from kernel context!", regs); + force_sig(SIGFPE); +out: + exception_exit(prev_state); +} + +asmlinkage void do_msa(struct pt_regs *regs) +{ + enum ctx_state prev_state; + int err; + + prev_state = exception_enter(); + + if (!cpu_has_msa || test_thread_flag(TIF_32BIT_FPREGS)) { + force_sig(SIGILL); goto out; } - force_sig(SIGILL, current); + die_if_kernel("do_msa invoked from kernel context!", regs); + err = enable_restore_fp_context(1); + if (err) + force_sig(SIGILL); out: exception_exit(prev_state); } @@ -1177,7 +1552,7 @@ asmlinkage void do_mdmx(struct pt_regs *regs) enum ctx_state prev_state; prev_state = exception_enter(); - force_sig(SIGILL, current); + force_sig(SIGILL); exception_exit(prev_state); } @@ -1187,16 +1562,13 @@ asmlinkage void do_mdmx(struct pt_regs *regs) asmlinkage void do_watch(struct pt_regs *regs) { enum ctx_state prev_state; - u32 cause; prev_state = exception_enter(); /* * Clear WP (bit 22) bit of cause register so we don't loop * forever. */ - cause = read_c0_cause(); - cause &= ~(1 << 22); - write_c0_cause(cause); + clear_c0_cause(CAUSEF_WP); /* * If the current thread has the watch registers loaded, save @@ -1206,7 +1578,7 @@ asmlinkage void do_watch(struct pt_regs *regs) if (test_tsk_thread_flag(current, TIF_LOAD_WATCH)) { mips_read_watch_registers(); local_irq_enable(); - force_sig(SIGTRAP, current); + force_sig_fault(SIGTRAP, TRAP_HWBKPT, NULL); } else { mips_clear_watch_registers(); local_irq_enable(); @@ -1216,7 +1588,6 @@ asmlinkage void do_watch(struct pt_regs *regs) asmlinkage void do_mcheck(struct pt_regs *regs) { - const int field = 2 * sizeof(unsigned long); int multi_match = regs->cp0_status & ST0_TS; enum ctx_state prev_state; @@ -1224,16 +1595,12 @@ asmlinkage void do_mcheck(struct pt_regs *regs) show_regs(regs); if (multi_match) { - printk("Index : %0x\n", read_c0_index()); - printk("Pagemask: %0x\n", read_c0_pagemask()); - printk("EntryHi : %0*lx\n", field, read_c0_entryhi()); - printk("EntryLo0: %0*lx\n", field, read_c0_entrylo0()); - printk("EntryLo1: %0*lx\n", field, read_c0_entrylo1()); - printk("\n"); + dump_tlb_regs(); + pr_info("\n"); dump_tlb_all(); } - show_code((unsigned int __user *) regs->cp0_epc); + show_code((void *)regs->cp0_epc, user_mode(regs)); /* * Some chips may have other causes of machine check (e.g. SB1 @@ -1242,7 +1609,6 @@ asmlinkage void do_mcheck(struct pt_regs *regs) panic("Caught Machine Check exception - %scaused by multiple " "matching entries in the TLB.", (multi_match) ? "" : "not "); - exception_exit(prev_state); } asmlinkage void do_mt(struct pt_regs *regs) @@ -1277,7 +1643,7 @@ asmlinkage void do_mt(struct pt_regs *regs) } die_if_kernel("MIPS MT Thread exception in kernel", regs); - force_sig(SIGILL, current); + force_sig(SIGILL); } @@ -1286,7 +1652,7 @@ asmlinkage void do_dsp(struct pt_regs *regs) if (cpu_has_dsp) panic("Unexpected DSP exception"); - force_sig(SIGILL, current); + force_sig(SIGILL); } asmlinkage void do_reserved(struct pt_regs *regs) @@ -1320,31 +1686,94 @@ __setup("nol2par", nol2parity); * Some MIPS CPUs can enable/disable for cache parity detection, but do * it different ways. */ -static inline void parity_protection_init(void) +static inline __init void parity_protection_init(void) { +#define ERRCTL_PE 0x80000000 +#define ERRCTL_L2P 0x00800000 + + if (mips_cm_revision() >= CM_REV_CM3) { + ulong gcr_ectl, cp0_ectl; + + /* + * With CM3 systems we need to ensure that the L1 & L2 + * parity enables are set to the same value, since this + * is presumed by the hardware engineers. + * + * If the user disabled either of L1 or L2 ECC checking, + * disable both. + */ + l1parity &= l2parity; + l2parity &= l1parity; + + /* Probe L1 ECC support */ + cp0_ectl = read_c0_errctl(); + write_c0_errctl(cp0_ectl | ERRCTL_PE); + back_to_back_c0_hazard(); + cp0_ectl = read_c0_errctl(); + + /* Probe L2 ECC support */ + gcr_ectl = read_gcr_err_control(); + + if (!(gcr_ectl & CM_GCR_ERR_CONTROL_L2_ECC_SUPPORT) || + !(cp0_ectl & ERRCTL_PE)) { + /* + * One of L1 or L2 ECC checking isn't supported, + * so we cannot enable either. + */ + l1parity = l2parity = 0; + } + + /* Configure L1 ECC checking */ + if (l1parity) + cp0_ectl |= ERRCTL_PE; + else + cp0_ectl &= ~ERRCTL_PE; + write_c0_errctl(cp0_ectl); + back_to_back_c0_hazard(); + WARN_ON(!!(read_c0_errctl() & ERRCTL_PE) != l1parity); + + /* Configure L2 ECC checking */ + if (l2parity) + gcr_ectl |= CM_GCR_ERR_CONTROL_L2_ECC_EN; + else + gcr_ectl &= ~CM_GCR_ERR_CONTROL_L2_ECC_EN; + write_gcr_err_control(gcr_ectl); + gcr_ectl = read_gcr_err_control(); + gcr_ectl &= CM_GCR_ERR_CONTROL_L2_ECC_EN; + WARN_ON(!!gcr_ectl != l2parity); + + pr_info("Cache parity protection %s\n", + str_enabled_disabled(l1parity)); + return; + } + switch (current_cpu_type()) { case CPU_24K: case CPU_34K: case CPU_74K: case CPU_1004K: + case CPU_1074K: + case CPU_INTERAPTIV: + case CPU_PROAPTIV: + case CPU_P5600: + case CPU_QEMU_GENERIC: + case CPU_P6600: { -#define ERRCTL_PE 0x80000000 -#define ERRCTL_L2P 0x00800000 unsigned long errctl; unsigned int l1parity_present, l2parity_present; - errctl = read_c0_ecc(); + errctl = read_c0_errctl(); errctl &= ~(ERRCTL_PE|ERRCTL_L2P); /* probe L1 parity support */ - write_c0_ecc(errctl | ERRCTL_PE); + write_c0_errctl(errctl | ERRCTL_PE); back_to_back_c0_hazard(); - l1parity_present = (read_c0_ecc() & ERRCTL_PE); + l1parity_present = (read_c0_errctl() & ERRCTL_PE); /* probe L2 parity support */ - write_c0_ecc(errctl|ERRCTL_L2P); + write_c0_errctl(errctl|ERRCTL_L2P); back_to_back_c0_hazard(); - l2parity_present = (read_c0_ecc() & ERRCTL_L2P); + l2parity_present = (read_c0_errctl() & ERRCTL_L2P); if (l1parity_present && l2parity_present) { if (l1parity) @@ -1363,32 +1792,32 @@ static inline void parity_protection_init(void) printk(KERN_INFO "Writing ErrCtl register=%08lx\n", errctl); - write_c0_ecc(errctl); + write_c0_errctl(errctl); back_to_back_c0_hazard(); - errctl = read_c0_ecc(); + errctl = read_c0_errctl(); printk(KERN_INFO "Readback ErrCtl register=%08lx\n", errctl); if (l1parity_present) - printk(KERN_INFO "Cache parity protection %sabled\n", - (errctl & ERRCTL_PE) ? "en" : "dis"); + pr_info("Cache parity protection %s\n", + str_enabled_disabled(errctl & ERRCTL_PE)); if (l2parity_present) { if (l1parity_present && l1parity) errctl ^= ERRCTL_L2P; - printk(KERN_INFO "L2 cache parity protection %sabled\n", - (errctl & ERRCTL_L2P) ? "en" : "dis"); + pr_info("L2 cache parity protection %s\n", + str_enabled_disabled(errctl & ERRCTL_L2P)); } } break; case CPU_5KC: case CPU_5KE: - case CPU_LOONGSON1: - write_c0_ecc(0x80000000); + case CPU_LOONGSON32: + write_c0_errctl(0x80000000); back_to_back_c0_hazard(); /* Set the PE bit (bit 31) in the c0_errctl register. */ - printk(KERN_INFO "Cache parity protection %sabled\n", - (read_c0_ecc() & 0x80000000) ? "en" : "dis"); + pr_info("Cache parity protection %s\n", + str_enabled_disabled(read_c0_errctl() & 0x80000000)); break; case CPU_20KC: case CPU_25KF: @@ -1416,14 +1845,27 @@ asmlinkage void cache_parity_error(void) printk("Decoded c0_cacheerr: %s cache fault in %s reference.\n", reg_val & (1<<30) ? "secondary" : "primary", reg_val & (1<<31) ? "data" : "insn"); - printk("Error bits: %s%s%s%s%s%s%s\n", - reg_val & (1<<29) ? "ED " : "", - reg_val & (1<<28) ? "ET " : "", - reg_val & (1<<26) ? "EE " : "", - reg_val & (1<<25) ? "EB " : "", - reg_val & (1<<24) ? "EI " : "", - reg_val & (1<<23) ? "E1 " : "", - reg_val & (1<<22) ? "E0 " : ""); + if ((cpu_has_mips_r2_r6) && + ((current_cpu_data.processor_id & 0xff0000) == PRID_COMP_MIPS)) { + pr_err("Error bits: %s%s%s%s%s%s%s%s\n", + reg_val & (1<<29) ? "ED " : "", + reg_val & (1<<28) ? "ET " : "", + reg_val & (1<<27) ? "ES " : "", + reg_val & (1<<26) ? "EE " : "", + reg_val & (1<<25) ? "EB " : "", + reg_val & (1<<24) ? "EI " : "", + reg_val & (1<<23) ? "E1 " : "", + reg_val & (1<<22) ? "E0 " : ""); + } else { + pr_err("Error bits: %s%s%s%s%s%s%s\n", + reg_val & (1<<29) ? "ED " : "", + reg_val & (1<<28) ? "ET " : "", + reg_val & (1<<26) ? "EE " : "", + reg_val & (1<<25) ? "EB " : "", + reg_val & (1<<24) ? "EI " : "", + reg_val & (1<<23) ? "E1 " : "", + reg_val & (1<<22) ? "E0 " : ""); + } printk("IDX: 0x%08x\n", reg_val & ((1<<22)-1)); #if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) @@ -1437,6 +1879,66 @@ asmlinkage void cache_parity_error(void) panic("Can't handle the cache error!"); } +asmlinkage void do_ftlb(void) +{ + const int field = 2 * sizeof(unsigned long); + unsigned int reg_val; + + /* For the moment, report the problem and hang. */ + if ((cpu_has_mips_r2_r6) && + (((current_cpu_data.processor_id & 0xff0000) == PRID_COMP_MIPS) || + ((current_cpu_data.processor_id & 0xff0000) == PRID_COMP_LOONGSON))) { + pr_err("FTLB error exception, cp0_errctl=0x%08x:\n", + read_c0_errctl()); + pr_err("cp0_errorepc == %0*lx\n", field, read_c0_errorepc()); + reg_val = read_c0_cacheerr(); + pr_err("c0_cacheerr == %08x\n", reg_val); + + if ((reg_val & 0xc0000000) == 0xc0000000) { + pr_err("Decoded c0_cacheerr: FTLB parity error\n"); + } else { + pr_err("Decoded c0_cacheerr: %s cache fault in %s reference.\n", + reg_val & (1<<30) ? "secondary" : "primary", + reg_val & (1<<31) ? "data" : "insn"); + } + } else { + pr_err("FTLB error exception\n"); + } + /* Just print the cacheerr bits for now */ + cache_parity_error(); +} + +asmlinkage void do_gsexc(struct pt_regs *regs, u32 diag1) +{ + u32 exccode = (diag1 & LOONGSON_DIAG1_EXCCODE) >> + LOONGSON_DIAG1_EXCCODE_SHIFT; + enum ctx_state prev_state; + + prev_state = exception_enter(); + + switch (exccode) { + case 0x08: + /* Undocumented exception, will trigger on certain + * also-undocumented instructions accessible from userspace. + * Processor state is not otherwise corrupted, but currently + * we don't know how to proceed. Maybe there is some + * undocumented control flag to enable the instructions? + */ + force_sig(SIGILL); + break; + + default: + /* None of the other exceptions, documented or not, have + * further details given; none are encountered in the wild + * either. Panic in case some of them turn out to be fatal. + */ + show_regs(regs); + panic("Unhandled Loongson exception - GSCause = %08x", diag1); + } + + exception_exit(prev_state); +} + /* * SDBBP EJTAG debug exception handler. * We skip the instruction and return to the next instruction. @@ -1488,18 +1990,34 @@ int register_nmi_notifier(struct notifier_block *nb) void __noreturn nmi_exception_handler(struct pt_regs *regs) { + char str[100]; + + nmi_enter(); raw_notifier_call_chain(&nmi_chain, 0, regs); bust_spinlocks(1); - printk("NMI taken!!!!\n"); - die("NMI", regs); + snprintf(str, 100, "CPU%d NMI taken, CP0_EPC=%lx\n", + smp_processor_id(), regs->cp0_epc); + regs->cp0_epc = read_c0_errorepc(); + die(str, regs); + nmi_exit(); } -#define VECTORSPACING 0x100 /* for EI/VI mode */ - unsigned long ebase; +EXPORT_SYMBOL_GPL(ebase); unsigned long exception_handlers[32]; unsigned long vi_handlers[64]; +void reserve_exception_space(phys_addr_t addr, unsigned long size) +{ + /* + * reserve exception space on CPUs other than CPU0 + * is too late, since memblock is unavailable when APs + * up + */ + if (smp_processor_id() == 0) + memblock_reserve(addr, size); +} + void __init *set_except_vector(int n, void *addr) { unsigned long handler = (unsigned long) addr; @@ -1525,13 +2043,12 @@ void __init *set_except_vector(int n, void *addr) unsigned long jump_mask = ~((1 << 28) - 1); #endif u32 *buf = (u32 *)(ebase + 0x200); - unsigned int k0 = 26; if ((handler & jump_mask) == ((ebase + 0x200) & jump_mask)) { uasm_i_j(&buf, handler & ~jump_mask); uasm_i_nop(&buf); } else { - UASM_i_LA(&buf, k0, handler); - uasm_i_jr(&buf, k0); + UASM_i_LA(&buf, GPR_K0, handler); + uasm_i_jr(&buf, GPR_K0); uasm_i_nop(&buf); } local_flush_icache_range(ebase + 0x200, (unsigned long)buf); @@ -1545,130 +2062,71 @@ static void do_default_vi(void) panic("Caught unexpected vectored interrupt."); } -static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs) +void *set_vi_handler(int n, vi_handler_t addr) { + extern const u8 except_vec_vi[]; + extern const u8 except_vec_vi_ori[], except_vec_vi_end[]; + extern const u8 skipover_except_vec_vi[]; unsigned long handler; unsigned long old_handler = vi_handlers[n]; int srssets = current_cpu_data.srsets; u16 *h; unsigned char *b; + const u8 *vec_start; + int ori_offset; + int handler_len; BUG_ON(!cpu_has_veic && !cpu_has_vint); - BUG_ON((n < 0) && (n > 9)); if (addr == NULL) { handler = (unsigned long) do_default_vi; - srs = 0; } else handler = (unsigned long) addr; vi_handlers[n] = handler; b = (unsigned char *)(ebase + 0x200 + n*VECTORSPACING); - if (srs >= srssets) - panic("Shadow register set %d not supported", srs); - if (cpu_has_veic) { if (board_bind_eic_interrupt) - board_bind_eic_interrupt(n, srs); + board_bind_eic_interrupt(n, 0); } else if (cpu_has_vint) { /* SRSMap is only defined if shadow sets are implemented */ if (srssets > 1) - change_c0_srsmap(0xf << n*4, srs << n*4); + change_c0_srsmap(0xf << n*4, 0 << n*4); } - if (srs == 0) { - /* - * If no shadow set is selected then use the default handler - * that does normal register saving and standard interrupt exit - */ - extern char except_vec_vi, except_vec_vi_lui; - extern char except_vec_vi_ori, except_vec_vi_end; - extern char rollback_except_vec_vi; - char *vec_start = using_rollback_handler() ? - &rollback_except_vec_vi : &except_vec_vi; -#ifdef CONFIG_MIPS_MT_SMTC - /* - * We need to provide the SMTC vectored interrupt handler - * not only with the address of the handler, but with the - * Status.IM bit to be masked before going there. - */ - extern char except_vec_vi_mori; + vec_start = using_skipover_handler() ? skipover_except_vec_vi : + except_vec_vi; #if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_BIG_ENDIAN) - const int mori_offset = &except_vec_vi_mori - vec_start + 2; + ori_offset = except_vec_vi_ori - vec_start + 2; #else - const int mori_offset = &except_vec_vi_mori - vec_start; + ori_offset = except_vec_vi_ori - vec_start; #endif -#endif /* CONFIG_MIPS_MT_SMTC */ -#if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_BIG_ENDIAN) - const int lui_offset = &except_vec_vi_lui - vec_start + 2; - const int ori_offset = &except_vec_vi_ori - vec_start + 2; -#else - const int lui_offset = &except_vec_vi_lui - vec_start; - const int ori_offset = &except_vec_vi_ori - vec_start; -#endif - const int handler_len = &except_vec_vi_end - vec_start; + handler_len = except_vec_vi_end - vec_start; - if (handler_len > VECTORSPACING) { - /* - * Sigh... panicing won't help as the console - * is probably not configured :( - */ - panic("VECTORSPACING too small"); - } - - set_handler(((unsigned long)b - ebase), vec_start, -#ifdef CONFIG_CPU_MICROMIPS - (handler_len - 1)); -#else - handler_len); -#endif -#ifdef CONFIG_MIPS_MT_SMTC - BUG_ON(n > 7); /* Vector index %d exceeds SMTC maximum. */ - - h = (u16 *)(b + mori_offset); - *h = (0x100 << n); -#endif /* CONFIG_MIPS_MT_SMTC */ - h = (u16 *)(b + lui_offset); - *h = (handler >> 16) & 0xffff; - h = (u16 *)(b + ori_offset); - *h = (handler & 0xffff); - local_flush_icache_range((unsigned long)b, - (unsigned long)(b+handler_len)); - } - else { + if (handler_len > VECTORSPACING) { /* - * In other cases jump directly to the interrupt handler. It - * is the handler's responsibility to save registers if required - * (eg hi/lo) and return from the exception using "eret". + * Sigh... panicing won't help as the console + * is probably not configured :( */ - u32 insn; + panic("VECTORSPACING too small"); + } - h = (u16 *)b; - /* j handler */ + set_handler(((unsigned long)b - ebase), vec_start, #ifdef CONFIG_CPU_MICROMIPS - insn = 0xd4000000 | (((u32)handler & 0x07ffffff) >> 1); + (handler_len - 1)); #else - insn = 0x08000000 | (((u32)handler & 0x0fffffff) >> 2); + handler_len); #endif - h[0] = (insn >> 16) & 0xffff; - h[1] = insn & 0xffff; - h[2] = 0; - h[3] = 0; - local_flush_icache_range((unsigned long)b, - (unsigned long)(b+8)); - } + /* insert offset into vi_handlers[] */ + h = (u16 *)(b + ori_offset); + *h = n * sizeof(handler); + local_flush_icache_range((unsigned long)b, + (unsigned long)(b+handler_len)); return (void *)old_handler; } -void *set_vi_handler(int n, vi_handler_t addr) -{ - return set_vi_srs_handler(n, addr, 0); -} - -extern void tlb_init(void); - /* * Timer interrupt */ @@ -1682,7 +2140,13 @@ int cp0_compare_irq_shift; int cp0_perfcount_irq; EXPORT_SYMBOL_GPL(cp0_perfcount_irq); -static int __cpuinitdata noulri; +/* + * Fast debug channel IRQ or -1 if not present + */ +int cp0_fdc_irq; +EXPORT_SYMBOL_GPL(cp0_fdc_irq); + +static int noulri; static int __init ulri_disable(char *s) { @@ -1693,32 +2157,16 @@ static int __init ulri_disable(char *s) } __setup("noulri", ulri_disable); -void __cpuinit per_cpu_trap_init(bool is_boot_cpu) +/* configure STATUS register */ +static void configure_status(void) { - unsigned int cpu = smp_processor_id(); - unsigned int status_set = ST0_CU0; - unsigned int hwrena = cpu_hwrena_impl_bits; -#ifdef CONFIG_MIPS_MT_SMTC - int secondaryTC = 0; - int bootTC = (cpu == 0); - - /* - * Only do per_cpu_trap_init() for first TC of Each VPE. - * Note that this hack assumes that the SMTC init code - * assigns TCs consecutively and in ascending order. - */ - - if (((read_c0_tcbind() & TCBIND_CURTC) != 0) && - ((read_c0_tcbind() & TCBIND_CURVPE) == cpu_data[cpu - 1].vpe_id)) - secondaryTC = 1; -#endif /* CONFIG_MIPS_MT_SMTC */ - /* * Disable coprocessors and select 32-bit or 64-bit addressing * and the 16/32 or 32/32 FPR register model. Reset the BEV * flag that some firmware may have left set and the TS bit (for * IP27). Set XX for ISA IV code to work. */ + unsigned int status_set = ST0_KERNEL_CUMASK; #ifdef CONFIG_64BIT status_set |= ST0_FR|ST0_KX|ST0_SX|ST0_UX; #endif @@ -1729,24 +2177,46 @@ void __cpuinit per_cpu_trap_init(bool is_boot_cpu) change_c0_status(ST0_CU|ST0_MX|ST0_RE|ST0_FR|ST0_BEV|ST0_TS|ST0_KX|ST0_SX|ST0_UX, status_set); + back_to_back_c0_hazard(); +} - if (cpu_has_mips_r2) - hwrena |= 0x0000000f; +unsigned int hwrena; +EXPORT_SYMBOL_GPL(hwrena); + +/* configure HWRENA register */ +static void configure_hwrena(void) +{ + hwrena = cpu_hwrena_impl_bits; + + if (cpu_has_mips_r2_r6) + hwrena |= MIPS_HWRENA_CPUNUM | + MIPS_HWRENA_SYNCISTEP | + MIPS_HWRENA_CC | + MIPS_HWRENA_CCRES; if (!noulri && cpu_has_userlocal) - hwrena |= (1 << 29); + hwrena |= MIPS_HWRENA_ULR; if (hwrena) write_c0_hwrena(hwrena); +} -#ifdef CONFIG_MIPS_MT_SMTC - if (!secondaryTC) { -#endif /* CONFIG_MIPS_MT_SMTC */ - - if (cpu_has_veic || cpu_has_vint) { +static void configure_exception_vector(void) +{ + if (cpu_has_mips_r2_r6) { unsigned long sr = set_c0_status(ST0_BEV); + /* If available, use WG to set top bits of EBASE */ + if (cpu_has_ebase_wg) { +#ifdef CONFIG_64BIT + write_c0_ebase_64(ebase | MIPS_EBASE_WG); +#else + write_c0_ebase(ebase | MIPS_EBASE_WG); +#endif + } write_c0_ebase(ebase); write_c0_status(sr); + } + if (cpu_has_veic || cpu_has_vint) { /* Setting vector spacing enables EI/VI mode */ change_c0_intctl(0x3e0, VECTORSPACING); } @@ -1758,59 +2228,58 @@ void __cpuinit per_cpu_trap_init(bool is_boot_cpu) } else set_c0_cause(CAUSEF_IV); } +} + +void per_cpu_trap_init(bool is_boot_cpu) +{ + unsigned int cpu = smp_processor_id(); + + configure_status(); + configure_hwrena(); + + configure_exception_vector(); /* * Before R2 both interrupt numbers were fixed to 7, so on R2 only: * * o read IntCtl.IPTI to determine the timer interrupt * o read IntCtl.IPPCI to determine the performance counter interrupt + * o read IntCtl.IPFDC to determine the fast debug channel interrupt */ - if (cpu_has_mips_r2) { + if (cpu_has_mips_r2_r6) { cp0_compare_irq_shift = CAUSEB_TI - CAUSEB_IP; cp0_compare_irq = (read_c0_intctl() >> INTCTLB_IPTI) & 7; cp0_perfcount_irq = (read_c0_intctl() >> INTCTLB_IPPCI) & 7; - if (cp0_perfcount_irq == cp0_compare_irq) - cp0_perfcount_irq = -1; + cp0_fdc_irq = (read_c0_intctl() >> INTCTLB_IPFDC) & 7; + if (!cp0_fdc_irq) + cp0_fdc_irq = -1; + } else { cp0_compare_irq = CP0_LEGACY_COMPARE_IRQ; cp0_compare_irq_shift = CP0_LEGACY_PERFCNT_IRQ; cp0_perfcount_irq = -1; + cp0_fdc_irq = -1; } -#ifdef CONFIG_MIPS_MT_SMTC - } -#endif /* CONFIG_MIPS_MT_SMTC */ - - if (!cpu_data[cpu].asid_cache) - cpu_data[cpu].asid_cache = ASID_FIRST_VERSION; + if (cpu_has_mmid) + cpu_data[cpu].asid_cache = 0; + else if (!cpu_data[cpu].asid_cache) + cpu_data[cpu].asid_cache = asid_first_version(cpu); - atomic_inc(&init_mm.mm_count); + mmgrab(&init_mm); current->active_mm = &init_mm; BUG_ON(current->mm); enter_lazy_tlb(&init_mm, current); -#ifdef CONFIG_MIPS_MT_SMTC - if (bootTC) { -#endif /* CONFIG_MIPS_MT_SMTC */ - /* Boot CPU's cache setup in setup_arch(). */ - if (!is_boot_cpu) - cpu_cache_init(); - tlb_init(); -#ifdef CONFIG_MIPS_MT_SMTC - } else if (!secondaryTC) { - /* - * First TC in non-boot VPE must do subset of tlb_init() - * for MMU countrol registers. - */ - write_c0_pagemask(PM_DEFAULT_MASK); - write_c0_wired(0); - } -#endif /* CONFIG_MIPS_MT_SMTC */ + /* Boot CPU's cache setup in setup_arch(). */ + if (!is_boot_cpu) + cpu_cache_init(); + tlb_init(); TLBMISS_HANDLER_SETUP(); } /* Install CPU exception handler */ -void __cpuinit set_handler(unsigned long offset, void *addr, unsigned long size) +void set_handler(unsigned long offset, const void *addr, unsigned long size) { #ifdef CONFIG_CPU_MICROMIPS memcpy((void *)(ebase + offset), ((unsigned char *)addr - 1), size); @@ -1820,18 +2289,18 @@ void __cpuinit set_handler(unsigned long offset, void *addr, unsigned long size) local_flush_icache_range(ebase + offset, ebase + offset + size); } -static char panic_null_cerr[] __cpuinitdata = - "Trying to set NULL cache error exception handler"; +static const char panic_null_cerr[] = + "Trying to set NULL cache error exception handler\n"; /* * Install uncached CPU exception handler. * This is suitable only for the cache error exception which is the only * exception handler that is being run uncached. */ -void __cpuinit set_uncached_handler(unsigned long offset, void *addr, +void set_uncached_handler(unsigned long offset, void *addr, unsigned long size) { - unsigned long uncached_ebase = CKSEG1ADDR(ebase); + unsigned long uncached_ebase = CKSEG1ADDR_OR_64BIT(__pa(ebase)); if (!addr) panic(panic_null_cerr); @@ -1853,28 +2322,43 @@ void __init trap_init(void) extern char except_vec3_generic; extern char except_vec4; extern char except_vec3_r4000; - unsigned long i; + unsigned long i, vec_size; + phys_addr_t ebase_pa; check_wait(); -#if defined(CONFIG_KGDB) - if (kgdb_early_setup) - return; /* Already done */ -#endif - - if (cpu_has_veic || cpu_has_vint) { - unsigned long size = 0x200 + VECTORSPACING*64; - ebase = (unsigned long) - __alloc_bootmem(size, 1 << fls(size), 0); + if (!cpu_has_mips_r2_r6) { + ebase = CAC_BASE; + vec_size = 0x400; } else { -#ifdef CONFIG_KVM_GUEST -#define KVM_GUEST_KSEG0 0x40000000 - ebase = KVM_GUEST_KSEG0; -#else - ebase = CKSEG0; -#endif - if (cpu_has_mips_r2) - ebase += (read_c0_ebase() & 0x3ffff000); + if (cpu_has_veic || cpu_has_vint) + vec_size = 0x200 + VECTORSPACING*64; + else + vec_size = PAGE_SIZE; + + ebase_pa = memblock_phys_alloc(vec_size, 1 << fls(vec_size)); + if (!ebase_pa) + panic("%s: Failed to allocate %lu bytes align=0x%x\n", + __func__, vec_size, 1 << fls(vec_size)); + + /* + * Try to ensure ebase resides in KSeg0 if possible. + * + * It shouldn't generally be in XKPhys on MIPS64 to avoid + * hitting a poorly defined exception base for Cache Errors. + * The allocation is likely to be in the low 512MB of physical, + * in which case we should be able to convert to KSeg0. + * + * EVA is special though as it allows segments to be rearranged + * and to become uncached during cache error handling. + */ + if (!IS_ENABLED(CONFIG_EVA) && ebase_pa < 0x20000000) + ebase = CKSEG0ADDR(ebase_pa); + else + ebase = (unsigned long)phys_to_virt(ebase_pa); + if (ebase_pa >= 0x20000000) + pr_warn("ebase(%pa) should better be in KSeg0", + &ebase_pa); } if (cpu_has_mmips) { @@ -1889,10 +2373,11 @@ void __init trap_init(void) if (board_ebase_setup) board_ebase_setup(); per_cpu_trap_init(true); + memblock_set_bottom_up(false); /* * Copy the generic exception handlers to their final destination. - * This will be overriden later as suitable for a particular + * This will be overridden later as suitable for a particular * configuration. */ set_handler(0x180, &except_vec3_generic, 0x80); @@ -1904,7 +2389,7 @@ void __init trap_init(void) set_except_vector(i, handle_reserved); /* - * Copy the EJTAG debug exception vector handler code to it's final + * Copy the EJTAG debug exception vector handler code to its final * destination. */ if (cpu_has_ejtag && board_ejtag_handler_setup) @@ -1914,7 +2399,7 @@ void __init trap_init(void) * Only some CPUs have the watch exceptions. */ if (cpu_has_watch) - set_except_vector(23, handle_watch); + set_except_vector(EXCCODE_WATCH, handle_watch); /* * Initialise interrupt handlers @@ -1941,57 +2426,64 @@ void __init trap_init(void) if (board_be_init) board_be_init(); - set_except_vector(0, using_rollback_handler() ? rollback_handle_int - : handle_int); - set_except_vector(1, handle_tlbm); - set_except_vector(2, handle_tlbl); - set_except_vector(3, handle_tlbs); - - set_except_vector(4, handle_adel); - set_except_vector(5, handle_ades); - - set_except_vector(6, handle_ibe); - set_except_vector(7, handle_dbe); - - set_except_vector(8, handle_sys); - set_except_vector(9, handle_bp); - set_except_vector(10, rdhwr_noopt ? handle_ri : - (cpu_has_vtag_icache ? - handle_ri_rdhwr_vivt : handle_ri_rdhwr)); - set_except_vector(11, handle_cpu); - set_except_vector(12, handle_ov); - set_except_vector(13, handle_tr); - - if (current_cpu_type() == CPU_R6000 || - current_cpu_type() == CPU_R6000A) { - /* - * The R6000 is the only R-series CPU that features a machine - * check exception (similar to the R4000 cache error) and - * unaligned ldc1/sdc1 exception. The handlers have not been - * written yet. Well, anyway there is no R6000 machine on the - * current list of targets for Linux/MIPS. - * (Duh, crap, there is someone with a triple R6k machine) - */ - //set_except_vector(14, handle_mc); - //set_except_vector(15, handle_ndc); + set_except_vector(EXCCODE_INT, using_skipover_handler() ? + skipover_handle_int : handle_int); + set_except_vector(EXCCODE_MOD, handle_tlbm); + set_except_vector(EXCCODE_TLBL, handle_tlbl); + set_except_vector(EXCCODE_TLBS, handle_tlbs); + + set_except_vector(EXCCODE_ADEL, handle_adel); + set_except_vector(EXCCODE_ADES, handle_ades); + + set_except_vector(EXCCODE_IBE, handle_ibe); + set_except_vector(EXCCODE_DBE, handle_dbe); + + set_except_vector(EXCCODE_SYS, handle_sys); + set_except_vector(EXCCODE_BP, handle_bp); + + if (rdhwr_noopt) + set_except_vector(EXCCODE_RI, handle_ri); + else { + if (cpu_has_vtag_icache) + set_except_vector(EXCCODE_RI, handle_ri_rdhwr_tlbp); + else if (current_cpu_type() == CPU_LOONGSON64) + set_except_vector(EXCCODE_RI, handle_ri_rdhwr_tlbp); + else + set_except_vector(EXCCODE_RI, handle_ri_rdhwr); } + set_except_vector(EXCCODE_CPU, handle_cpu); + set_except_vector(EXCCODE_OV, handle_ov); + set_except_vector(EXCCODE_TR, handle_tr); + set_except_vector(EXCCODE_MSAFPE, handle_msa_fpe); if (board_nmi_handler_setup) board_nmi_handler_setup(); if (cpu_has_fpu && !cpu_has_nofpuex) - set_except_vector(15, handle_fpe); + set_except_vector(EXCCODE_FPE, handle_fpe); - set_except_vector(22, handle_mdmx); + if (cpu_has_ftlbparex) + set_except_vector(MIPS_EXCCODE_TLBPAR, handle_ftlb); + + if (cpu_has_gsexcex) + set_except_vector(LOONGSON_EXCCODE_GSEXC, handle_gsexc); + + if (cpu_has_rixiex) { + set_except_vector(EXCCODE_TLBRI, tlb_do_page_fault_0); + set_except_vector(EXCCODE_TLBXI, tlb_do_page_fault_0); + } + + set_except_vector(EXCCODE_MSADIS, handle_msa); + set_except_vector(EXCCODE_MDMX, handle_mdmx); if (cpu_has_mcheck) - set_except_vector(24, handle_mcheck); + set_except_vector(EXCCODE_MCHECK, handle_mcheck); if (cpu_has_mipsmt) - set_except_vector(25, handle_mt); + set_except_vector(EXCCODE_THREAD, handle_mt); - set_except_vector(26, handle_dsp); + set_except_vector(EXCCODE_DSPDIS, handle_dsp); if (board_cache_error_setup) board_cache_error_setup(); @@ -2004,9 +2496,38 @@ void __init trap_init(void) else set_handler(0x080, &except_vec3_generic, 0x80); - local_flush_icache_range(ebase, ebase + 0x400); + local_flush_icache_range(ebase, ebase + vec_size); sort_extable(__start___dbe_table, __stop___dbe_table); cu2_notifier(default_cu2_call, 0x80000000); /* Run last */ } + +static int trap_pm_notifier(struct notifier_block *self, unsigned long cmd, + void *v) +{ + switch (cmd) { + case CPU_PM_ENTER_FAILED: + case CPU_PM_EXIT: + configure_status(); + configure_hwrena(); + configure_exception_vector(); + + /* Restore register with CPU number for TLB handlers */ + TLBMISS_HANDLER_RESTORE(); + + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block trap_pm_notifier_block = { + .notifier_call = trap_pm_notifier, +}; + +static int __init trap_pm_init(void) +{ + return cpu_pm_register_notifier(&trap_pm_notifier_block); +} +arch_initcall(trap_pm_init); diff --git a/arch/mips/kernel/unaligned.c b/arch/mips/kernel/unaligned.c index c369a5d35527..db652c99b72e 100644 --- a/arch/mips/kernel/unaligned.c +++ b/arch/mips/kernel/unaligned.c @@ -7,6 +7,7 @@ * * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle * Copyright (C) 1999 Silicon Graphics, Inc. + * Copyright (C) 2014 Imagination Technologies Ltd. * * This file contains exception handler for address error exception with the * special capability to execute faulting instructions in software. The @@ -84,15 +85,16 @@ #include <asm/branch.h> #include <asm/byteorder.h> #include <asm/cop2.h> +#include <asm/debug.h> #include <asm/fpu.h> #include <asm/fpu_emulator.h> #include <asm/inst.h> -#include <asm/uaccess.h> -#include <asm/fpu.h> -#include <asm/fpu_emulator.h> +#include <asm/unaligned-emul.h> +#include <asm/mmu_context.h> +#include <asm/traps.h> +#include <linux/uaccess.h> -#define STR(x) __STR(x) -#define __STR(x) #x +#include "access-helper.h" enum { UNALIGNED_ACTION_QUIET, @@ -107,329 +109,13 @@ static u32 unaligned_action; #endif extern void show_registers(struct pt_regs *regs); -#ifdef __BIG_ENDIAN -#define LoadHW(addr, value, res) \ - __asm__ __volatile__ (".set\tnoat\n" \ - "1:\tlb\t%0, 0(%2)\n" \ - "2:\tlbu\t$1, 1(%2)\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - "3:\t.set\tat\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); - -#define LoadW(addr, value, res) \ - __asm__ __volatile__ ( \ - "1:\tlwl\t%0, (%2)\n" \ - "2:\tlwr\t%0, 3(%2)\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); - -#define LoadHWU(addr, value, res) \ - __asm__ __volatile__ ( \ - ".set\tnoat\n" \ - "1:\tlbu\t%0, 0(%2)\n" \ - "2:\tlbu\t$1, 1(%2)\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".set\tat\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); - -#define LoadWU(addr, value, res) \ - __asm__ __volatile__ ( \ - "1:\tlwl\t%0, (%2)\n" \ - "2:\tlwr\t%0, 3(%2)\n\t" \ - "dsll\t%0, %0, 32\n\t" \ - "dsrl\t%0, %0, 32\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - "\t.section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); - -#define LoadDW(addr, value, res) \ - __asm__ __volatile__ ( \ - "1:\tldl\t%0, (%2)\n" \ - "2:\tldr\t%0, 7(%2)\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - "\t.section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); - -#define StoreHW(addr, value, res) \ - __asm__ __volatile__ ( \ - ".set\tnoat\n" \ - "1:\tsb\t%1, 1(%2)\n\t" \ - "srl\t$1, %1, 0x8\n" \ - "2:\tsb\t$1, 0(%2)\n\t" \ - ".set\tat\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); - -#define StoreW(addr, value, res) \ - __asm__ __volatile__ ( \ - "1:\tswl\t%1,(%2)\n" \ - "2:\tswr\t%1, 3(%2)\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); - -#define StoreDW(addr, value, res) \ - __asm__ __volatile__ ( \ - "1:\tsdl\t%1,(%2)\n" \ - "2:\tsdr\t%1, 7(%2)\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); -#endif - -#ifdef __LITTLE_ENDIAN -#define LoadHW(addr, value, res) \ - __asm__ __volatile__ (".set\tnoat\n" \ - "1:\tlb\t%0, 1(%2)\n" \ - "2:\tlbu\t$1, 0(%2)\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - "3:\t.set\tat\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); - -#define LoadW(addr, value, res) \ - __asm__ __volatile__ ( \ - "1:\tlwl\t%0, 3(%2)\n" \ - "2:\tlwr\t%0, (%2)\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); - -#define LoadHWU(addr, value, res) \ - __asm__ __volatile__ ( \ - ".set\tnoat\n" \ - "1:\tlbu\t%0, 1(%2)\n" \ - "2:\tlbu\t$1, 0(%2)\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".set\tat\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); - -#define LoadWU(addr, value, res) \ - __asm__ __volatile__ ( \ - "1:\tlwl\t%0, 3(%2)\n" \ - "2:\tlwr\t%0, (%2)\n\t" \ - "dsll\t%0, %0, 32\n\t" \ - "dsrl\t%0, %0, 32\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - "\t.section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); - -#define LoadDW(addr, value, res) \ - __asm__ __volatile__ ( \ - "1:\tldl\t%0, 7(%2)\n" \ - "2:\tldr\t%0, (%2)\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - "\t.section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); - -#define StoreHW(addr, value, res) \ - __asm__ __volatile__ ( \ - ".set\tnoat\n" \ - "1:\tsb\t%1, 0(%2)\n\t" \ - "srl\t$1,%1, 0x8\n" \ - "2:\tsb\t$1, 1(%2)\n\t" \ - ".set\tat\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); - -#define StoreW(addr, value, res) \ - __asm__ __volatile__ ( \ - "1:\tswl\t%1, 3(%2)\n" \ - "2:\tswr\t%1, (%2)\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); - -#define StoreDW(addr, value, res) \ - __asm__ __volatile__ ( \ - "1:\tsdl\t%1, 7(%2)\n" \ - "2:\tsdr\t%1, (%2)\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); -#endif - static void emulate_load_store_insn(struct pt_regs *regs, - void __user *addr, unsigned int __user *pc) + void __user *addr, unsigned int *pc) { + unsigned long origpc, orig31, value; union mips_instruction insn; - unsigned long value; unsigned int res; - unsigned long origpc; - unsigned long orig31; - void __user *fault_addr = NULL; + bool user = user_mode(regs); origpc = (unsigned long)pc; orig31 = regs->regs[31]; @@ -439,7 +125,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, /* * This load never faults. */ - __get_user(insn.word, pc); + __get_inst32(&insn.word, pc, user); switch (insn.i_format.opcode) { /* @@ -475,11 +161,140 @@ static void emulate_load_store_insn(struct pt_regs *regs, * The remaining opcodes are the ones that are really of * interest. */ +#ifdef CONFIG_MACH_INGENIC + case spec2_op: + if (insn.mxu_lx_format.func != mxu_lx_op) + goto sigbus; /* other MXU instructions we don't care */ + + switch (insn.mxu_lx_format.op) { + case mxu_lxw_op: + if (user && !access_ok(addr, 4)) + goto sigbus; + LoadW(addr, value, res); + if (res) + goto fault; + compute_return_epc(regs); + regs->regs[insn.mxu_lx_format.rd] = value; + break; + case mxu_lxh_op: + if (user && !access_ok(addr, 2)) + goto sigbus; + LoadHW(addr, value, res); + if (res) + goto fault; + compute_return_epc(regs); + regs->regs[insn.dsp_format.rd] = value; + break; + case mxu_lxhu_op: + if (user && !access_ok(addr, 2)) + goto sigbus; + LoadHWU(addr, value, res); + if (res) + goto fault; + compute_return_epc(regs); + regs->regs[insn.dsp_format.rd] = value; + break; + case mxu_lxb_op: + case mxu_lxbu_op: + goto sigbus; + default: + goto sigill; + } + break; +#endif + case spec3_op: + if (insn.dsp_format.func == lx_op) { + switch (insn.dsp_format.op) { + case lwx_op: + if (user && !access_ok(addr, 4)) + goto sigbus; + LoadW(addr, value, res); + if (res) + goto fault; + compute_return_epc(regs); + regs->regs[insn.dsp_format.rd] = value; + break; + case lhx_op: + if (user && !access_ok(addr, 2)) + goto sigbus; + LoadHW(addr, value, res); + if (res) + goto fault; + compute_return_epc(regs); + regs->regs[insn.dsp_format.rd] = value; + break; + default: + goto sigill; + } + } +#ifdef CONFIG_EVA + else { + /* + * we can land here only from kernel accessing user + * memory, so we need to "switch" the address limit to + * user space, so that address check can work properly. + */ + switch (insn.spec3_format.func) { + case lhe_op: + if (!access_ok(addr, 2)) + goto sigbus; + LoadHWE(addr, value, res); + if (res) + goto fault; + compute_return_epc(regs); + regs->regs[insn.spec3_format.rt] = value; + break; + case lwe_op: + if (!access_ok(addr, 4)) + goto sigbus; + LoadWE(addr, value, res); + if (res) + goto fault; + compute_return_epc(regs); + regs->regs[insn.spec3_format.rt] = value; + break; + case lhue_op: + if (!access_ok(addr, 2)) + goto sigbus; + LoadHWUE(addr, value, res); + if (res) + goto fault; + compute_return_epc(regs); + regs->regs[insn.spec3_format.rt] = value; + break; + case she_op: + if (!access_ok(addr, 2)) + goto sigbus; + compute_return_epc(regs); + value = regs->regs[insn.spec3_format.rt]; + StoreHWE(addr, value, res); + if (res) + goto fault; + break; + case swe_op: + if (!access_ok(addr, 4)) + goto sigbus; + compute_return_epc(regs); + value = regs->regs[insn.spec3_format.rt]; + StoreWE(addr, value, res); + if (res) + goto fault; + break; + default: + goto sigill; + } + } +#endif + break; case lh_op: - if (!access_ok(VERIFY_READ, addr, 2)) + if (user && !access_ok(addr, 2)) goto sigbus; - LoadHW(addr, value, res); + if (IS_ENABLED(CONFIG_EVA) && user) + LoadHWE(addr, value, res); + else + LoadHW(addr, value, res); + if (res) goto fault; compute_return_epc(regs); @@ -487,10 +302,14 @@ static void emulate_load_store_insn(struct pt_regs *regs, break; case lw_op: - if (!access_ok(VERIFY_READ, addr, 4)) + if (user && !access_ok(addr, 4)) goto sigbus; - LoadW(addr, value, res); + if (IS_ENABLED(CONFIG_EVA) && user) + LoadWE(addr, value, res); + else + LoadW(addr, value, res); + if (res) goto fault; compute_return_epc(regs); @@ -498,10 +317,14 @@ static void emulate_load_store_insn(struct pt_regs *regs, break; case lhu_op: - if (!access_ok(VERIFY_READ, addr, 2)) + if (user && !access_ok(addr, 2)) goto sigbus; - LoadHWU(addr, value, res); + if (IS_ENABLED(CONFIG_EVA) && user) + LoadHWUE(addr, value, res); + else + LoadHWU(addr, value, res); + if (res) goto fault; compute_return_epc(regs); @@ -517,7 +340,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_READ, addr, 4)) + if (user && !access_ok(addr, 4)) goto sigbus; LoadWU(addr, value, res); @@ -540,7 +363,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_READ, addr, 8)) + if (user && !access_ok(addr, 8)) goto sigbus; LoadDW(addr, value, res); @@ -555,23 +378,33 @@ static void emulate_load_store_insn(struct pt_regs *regs, goto sigill; case sh_op: - if (!access_ok(VERIFY_WRITE, addr, 2)) + if (user && !access_ok(addr, 2)) goto sigbus; compute_return_epc(regs); value = regs->regs[insn.i_format.rt]; - StoreHW(addr, value, res); + + if (IS_ENABLED(CONFIG_EVA) && user) + StoreHWE(addr, value, res); + else + StoreHW(addr, value, res); + if (res) goto fault; break; case sw_op: - if (!access_ok(VERIFY_WRITE, addr, 4)) + if (user && !access_ok(addr, 4)) goto sigbus; compute_return_epc(regs); value = regs->regs[insn.i_format.rt]; - StoreW(addr, value, res); + + if (IS_ENABLED(CONFIG_EVA) && user) + StoreWE(addr, value, res); + else + StoreW(addr, value, res); + if (res) goto fault; break; @@ -585,7 +418,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_WRITE, addr, 8)) + if (user && !access_ok(addr, 8)) goto sigbus; compute_return_epc(regs); @@ -599,30 +432,124 @@ static void emulate_load_store_insn(struct pt_regs *regs, /* Cannot handle 64-bit instructions in 32-bit kernel */ goto sigill; +#ifdef CONFIG_MIPS_FP_SUPPORT + case lwc1_op: case ldc1_op: case swc1_op: case sdc1_op: + case cop1x_op: { + void __user *fault_addr = NULL; + die_if_kernel("Unaligned FP access in kernel code", regs); BUG_ON(!used_math()); - BUG_ON(!is_fpu_owner()); - lose_fpu(1); /* Save FPU state for the emulator. */ res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, &fault_addr); own_fpu(1); /* Restore FPU state. */ /* Signal if something went wrong. */ - process_fpemu_return(res, fault_addr); + process_fpemu_return(res, fault_addr, 0); if (res == 0) break; return; + } +#endif /* CONFIG_MIPS_FP_SUPPORT */ + +#ifdef CONFIG_CPU_HAS_MSA + + case msa_op: { + unsigned int wd, preempted; + enum msa_2b_fmt df; + union fpureg *fpr; + + if (!cpu_has_msa) + goto sigill; + + /* + * If we've reached this point then userland should have taken + * the MSA disabled exception & initialised vector context at + * some point in the past. + */ + BUG_ON(!thread_msa_context_live()); + + df = insn.msa_mi10_format.df; + wd = insn.msa_mi10_format.wd; + fpr = ¤t->thread.fpu.fpr[wd]; + + switch (insn.msa_mi10_format.func) { + case msa_ld_op: + if (!access_ok(addr, sizeof(*fpr))) + goto sigbus; + + do { + /* + * If we have live MSA context keep track of + * whether we get preempted in order to avoid + * the register context we load being clobbered + * by the live context as it's saved during + * preemption. If we don't have live context + * then it can't be saved to clobber the value + * we load. + */ + preempted = test_thread_flag(TIF_USEDMSA); + + res = __copy_from_user_inatomic(fpr, addr, + sizeof(*fpr)); + if (res) + goto fault; + + /* + * Update the hardware register if it is in use + * by the task in this quantum, in order to + * avoid having to save & restore the whole + * vector context. + */ + preempt_disable(); + if (test_thread_flag(TIF_USEDMSA)) { + write_msa_wr(wd, fpr, df); + preempted = 0; + } + preempt_enable(); + } while (preempted); + break; + + case msa_st_op: + if (!access_ok(addr, sizeof(*fpr))) + goto sigbus; + + /* + * Update from the hardware register if it is in use by + * the task in this quantum, in order to avoid having to + * save & restore the whole vector context. + */ + preempt_disable(); + if (test_thread_flag(TIF_USEDMSA)) + read_msa_wr(wd, fpr, df); + preempt_enable(); + + res = __copy_to_user_inatomic(addr, fpr, sizeof(*fpr)); + if (res) + goto fault; + break; + + default: + goto sigbus; + } + compute_return_epc(regs); + break; + } +#endif /* CONFIG_CPU_HAS_MSA */ + +#ifndef CONFIG_CPU_MIPSR6 /* * COP2 is available to implementor for application specific use. * It's up to applications to register a notifier chain and do * whatever they have to do, including possible sending of signals. + * + * This instruction has been reallocated in Release 6 */ case lwc2_op: cu2_notifier_call_chain(CU2_LWC2_OP, regs); @@ -639,7 +566,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, case sdc2_op: cu2_notifier_call_chain(CU2_SDC2_OP, regs); break; - +#endif default: /* * Pheeee... We encountered an yet unknown instruction or @@ -663,27 +590,27 @@ fault: return; die_if_kernel("Unhandled kernel unaligned access", regs); - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); return; sigbus: die_if_kernel("Unhandled kernel unaligned access", regs); - force_sig(SIGBUS, current); + force_sig(SIGBUS); return; sigill: die_if_kernel ("Unhandled kernel unaligned access or invalid instruction", regs); - force_sig(SIGILL, current); + force_sig(SIGILL); } /* Recode table from 16-bit register notation to 32-bit GPR. */ const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 }; /* Recode table from 16-bit STORE register notation to 32-bit GPR. */ -const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 }; +static const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 }; static void emulate_load_store_microMIPS(struct pt_regs *regs, void __user *addr) @@ -699,7 +626,7 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, unsigned long origpc, contpc; union mips_instruction insn; struct mm_decoded_insn mminsn; - void __user *fault_addr = NULL; + bool user = user_mode(regs); origpc = regs->cp0_epc; orig31 = regs->regs[31]; @@ -763,7 +690,7 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if (reg == 31) goto sigbus; - if (!access_ok(VERIFY_READ, addr, 8)) + if (user && !access_ok(addr, 8)) goto sigbus; LoadW(addr, value, res); @@ -782,7 +709,7 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if (reg == 31) goto sigbus; - if (!access_ok(VERIFY_WRITE, addr, 8)) + if (user && !access_ok(addr, 8)) goto sigbus; value = regs->regs[reg]; @@ -802,7 +729,7 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if (reg == 31) goto sigbus; - if (!access_ok(VERIFY_READ, addr, 16)) + if (user && !access_ok(addr, 16)) goto sigbus; LoadDW(addr, value, res); @@ -825,7 +752,7 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if (reg == 31) goto sigbus; - if (!access_ok(VERIFY_WRITE, addr, 16)) + if (user && !access_ok(addr, 16)) goto sigbus; value = regs->regs[reg]; @@ -848,11 +775,10 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if ((rvar > 9) || !reg) goto sigill; if (reg & 0x10) { - if (!access_ok - (VERIFY_READ, addr, 4 * (rvar + 1))) + if (user && !access_ok(addr, 4 * (rvar + 1))) goto sigbus; } else { - if (!access_ok(VERIFY_READ, addr, 4 * rvar)) + if (user && !access_ok(addr, 4 * rvar)) goto sigbus; } if (rvar == 9) @@ -885,11 +811,10 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if ((rvar > 9) || !reg) goto sigill; if (reg & 0x10) { - if (!access_ok - (VERIFY_WRITE, addr, 4 * (rvar + 1))) + if (user && !access_ok(addr, 4 * (rvar + 1))) goto sigbus; } else { - if (!access_ok(VERIFY_WRITE, addr, 4 * rvar)) + if (user && !access_ok(addr, 4 * rvar)) goto sigbus; } if (rvar == 9) @@ -923,11 +848,10 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if ((rvar > 9) || !reg) goto sigill; if (reg & 0x10) { - if (!access_ok - (VERIFY_READ, addr, 8 * (rvar + 1))) + if (user && !access_ok(addr, 8 * (rvar + 1))) goto sigbus; } else { - if (!access_ok(VERIFY_READ, addr, 8 * rvar)) + if (user && !access_ok(addr, 8 * rvar)) goto sigbus; } if (rvar == 9) @@ -965,11 +889,10 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, if ((rvar > 9) || !reg) goto sigill; if (reg & 0x10) { - if (!access_ok - (VERIFY_WRITE, addr, 8 * (rvar + 1))) + if (user && !access_ok(addr, 8 * (rvar + 1))) goto sigbus; } else { - if (!access_ok(VERIFY_WRITE, addr, 8 * rvar)) + if (user && !access_ok(addr, 8 * rvar)) goto sigbus; } if (rvar == 9) @@ -1015,6 +938,7 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, /* LL,SC,LLD,SCD are not serviced */ goto sigbus; +#ifdef CONFIG_MIPS_FP_SUPPORT case mm_pool32f_op: switch (insn.mm_x_format.func) { case mm_lwxc1_func: @@ -1029,7 +953,9 @@ static void emulate_load_store_microMIPS(struct pt_regs *regs, case mm_ldc132_op: case mm_sdc132_op: case mm_lwc132_op: - case mm_swc132_op: + case mm_swc132_op: { + void __user *fault_addr = NULL; + fpu_emul: /* roll back jump/branch */ regs->cp0_epc = origpc; @@ -1039,17 +965,18 @@ fpu_emul: BUG_ON(!used_math()); BUG_ON(!is_fpu_owner()); - lose_fpu(1); /* save the FPU state for the emulator */ res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, &fault_addr); own_fpu(1); /* restore FPU state */ /* If something went wrong, signal */ - process_fpemu_return(res, fault_addr); + process_fpemu_return(res, fault_addr, 0); if (res == 0) goto success; return; + } +#endif /* CONFIG_MIPS_FP_SUPPORT */ case mm_lh32_op: reg = insn.mm_i_format.rt; @@ -1084,7 +1011,7 @@ fpu_emul: case mm_lwm16_op: reg = insn.mm16_m_format.rlist; rvar = reg + 1; - if (!access_ok(VERIFY_READ, addr, 4 * rvar)) + if (user && !access_ok(addr, 4 * rvar)) goto sigbus; for (i = 16; rvar; rvar--, i++) { @@ -1104,7 +1031,7 @@ fpu_emul: case mm_swm16_op: reg = insn.mm16_m_format.rlist; rvar = reg + 1; - if (!access_ok(VERIFY_WRITE, addr, 4 * rvar)) + if (user && !access_ok(addr, 4 * rvar)) goto sigbus; for (i = 16; rvar; rvar--, i++) { @@ -1158,7 +1085,7 @@ fpu_emul: } loadHW: - if (!access_ok(VERIFY_READ, addr, 2)) + if (user && !access_ok(addr, 2)) goto sigbus; LoadHW(addr, value, res); @@ -1168,7 +1095,7 @@ loadHW: goto success; loadHWU: - if (!access_ok(VERIFY_READ, addr, 2)) + if (user && !access_ok(addr, 2)) goto sigbus; LoadHWU(addr, value, res); @@ -1178,7 +1105,7 @@ loadHWU: goto success; loadW: - if (!access_ok(VERIFY_READ, addr, 4)) + if (user && !access_ok(addr, 4)) goto sigbus; LoadW(addr, value, res); @@ -1196,7 +1123,7 @@ loadWU: * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_READ, addr, 4)) + if (user && !access_ok(addr, 4)) goto sigbus; LoadWU(addr, value, res); @@ -1218,7 +1145,7 @@ loadDW: * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_READ, addr, 8)) + if (user && !access_ok(addr, 8)) goto sigbus; LoadDW(addr, value, res); @@ -1232,7 +1159,7 @@ loadDW: goto sigill; storeHW: - if (!access_ok(VERIFY_WRITE, addr, 2)) + if (user && !access_ok(addr, 2)) goto sigbus; value = regs->regs[reg]; @@ -1242,7 +1169,7 @@ storeHW: goto success; storeW: - if (!access_ok(VERIFY_WRITE, addr, 4)) + if (user && !access_ok(addr, 4)) goto sigbus; value = regs->regs[reg]; @@ -1260,7 +1187,7 @@ storeDW: * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_WRITE, addr, 8)) + if (user && !access_ok(addr, 8)) goto sigbus; value = regs->regs[reg]; @@ -1290,20 +1217,20 @@ fault: return; die_if_kernel("Unhandled kernel unaligned access", regs); - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); return; sigbus: die_if_kernel("Unhandled kernel unaligned access", regs); - force_sig(SIGBUS, current); + force_sig(SIGBUS); return; sigill: die_if_kernel ("Unhandled kernel unaligned access or invalid instruction", regs); - force_sig(SIGILL, current); + force_sig(SIGILL); } static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) @@ -1315,6 +1242,9 @@ static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) u16 __user *pc16; unsigned long origpc; union mips16e_instruction mips16inst, oldinst; + unsigned int opcode; + int extended = 0; + bool user = user_mode(regs); origpc = regs->cp0_epc; orig31 = regs->regs[31]; @@ -1327,6 +1257,7 @@ static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) /* skip EXTEND instruction */ if (mips16inst.ri.opcode == MIPS16e_extend_op) { + extended = 1; pc16++; __get_user(mips16inst.full, pc16); } else if (delay_slot(regs)) { @@ -1339,7 +1270,8 @@ static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) goto sigbus; } - switch (mips16inst.ri.opcode) { + opcode = mips16inst.ri.opcode; + switch (opcode) { case MIPS16e_i64_op: /* I64 or RI64 instruction */ switch (mips16inst.i64.func) { /* I64/RI64 func field check */ case MIPS16e_ldpc_func: @@ -1359,9 +1291,40 @@ static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) goto sigbus; case MIPS16e_swsp_op: + reg = reg16to32[mips16inst.ri.rx]; + if (extended && cpu_has_mips16e2) + switch (mips16inst.ri.imm >> 5) { + case 0: /* SWSP */ + case 1: /* SWGP */ + break; + case 2: /* SHGP */ + opcode = MIPS16e_sh_op; + break; + default: + goto sigbus; + } + break; + case MIPS16e_lwpc_op: + reg = reg16to32[mips16inst.ri.rx]; + break; + case MIPS16e_lwsp_op: reg = reg16to32[mips16inst.ri.rx]; + if (extended && cpu_has_mips16e2) + switch (mips16inst.ri.imm >> 5) { + case 0: /* LWSP */ + case 1: /* LWGP */ + break; + case 2: /* LHGP */ + opcode = MIPS16e_lh_op; + break; + case 4: /* LHUGP */ + opcode = MIPS16e_lhu_op; + break; + default: + goto sigbus; + } break; case MIPS16e_i8_op: @@ -1375,7 +1338,7 @@ static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) break; } - switch (mips16inst.ri.opcode) { + switch (opcode) { case MIPS16e_lb_op: case MIPS16e_lbu_op: @@ -1383,7 +1346,7 @@ static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) goto sigbus; case MIPS16e_lh_op: - if (!access_ok(VERIFY_READ, addr, 2)) + if (user && !access_ok(addr, 2)) goto sigbus; LoadHW(addr, value, res); @@ -1394,7 +1357,7 @@ static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) break; case MIPS16e_lhu_op: - if (!access_ok(VERIFY_READ, addr, 2)) + if (user && !access_ok(addr, 2)) goto sigbus; LoadHWU(addr, value, res); @@ -1407,7 +1370,7 @@ static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) case MIPS16e_lw_op: case MIPS16e_lwpc_op: case MIPS16e_lwsp_op: - if (!access_ok(VERIFY_READ, addr, 4)) + if (user && !access_ok(addr, 4)) goto sigbus; LoadW(addr, value, res); @@ -1426,7 +1389,7 @@ static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_READ, addr, 4)) + if (user && !access_ok(addr, 4)) goto sigbus; LoadWU(addr, value, res); @@ -1450,7 +1413,7 @@ loadDW: * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_READ, addr, 8)) + if (user && !access_ok(addr, 8)) goto sigbus; LoadDW(addr, value, res); @@ -1465,7 +1428,7 @@ loadDW: goto sigill; case MIPS16e_sh_op: - if (!access_ok(VERIFY_WRITE, addr, 2)) + if (user && !access_ok(addr, 2)) goto sigbus; MIPS16e_compute_return_epc(regs, &oldinst); @@ -1478,7 +1441,7 @@ loadDW: case MIPS16e_sw_op: case MIPS16e_swsp_op: case MIPS16e_i8_op: /* actually - MIPS16e_swrasp_func */ - if (!access_ok(VERIFY_WRITE, addr, 4)) + if (user && !access_ok(addr, 4)) goto sigbus; MIPS16e_compute_return_epc(regs, &oldinst); @@ -1498,7 +1461,7 @@ writeDW: * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ - if (!access_ok(VERIFY_WRITE, addr, 8)) + if (user && !access_ok(addr, 8)) goto sigbus; MIPS16e_compute_return_epc(regs, &oldinst); @@ -1535,31 +1498,47 @@ fault: return; die_if_kernel("Unhandled kernel unaligned access", regs); - force_sig(SIGSEGV, current); + force_sig(SIGSEGV); return; sigbus: die_if_kernel("Unhandled kernel unaligned access", regs); - force_sig(SIGBUS, current); + force_sig(SIGBUS); return; sigill: die_if_kernel ("Unhandled kernel unaligned access or invalid instruction", regs); - force_sig(SIGILL, current); + force_sig(SIGILL); } asmlinkage void do_ade(struct pt_regs *regs) { enum ctx_state prev_state; - unsigned int __user *pc; - mm_segment_t seg; + unsigned int *pc; prev_state = exception_enter(); perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, regs->cp0_badvaddr); + +#ifdef CONFIG_64BIT + /* + * check, if we are hitting space between CPU implemented maximum + * virtual user address and 64bit maximum virtual user address + * and do exception handling to get EFAULTs for get_user/put_user + */ + if ((regs->cp0_badvaddr >= (1UL << cpu_vmbits)) && + (regs->cp0_badvaddr < XKSSEG)) { + if (fixup_exception(regs)) { + current->thread.cp0_baduaddr = regs->cp0_badvaddr; + return; + } + goto sigbus; + } +#endif + /* * Did we catch a fault trying to load an instruction? */ @@ -1590,45 +1569,31 @@ asmlinkage void do_ade(struct pt_regs *regs) show_registers(regs); if (cpu_has_mmips) { - seg = get_fs(); - if (!user_mode(regs)) - set_fs(KERNEL_DS); emulate_load_store_microMIPS(regs, (void __user *)regs->cp0_badvaddr); - set_fs(seg); - return; } if (cpu_has_mips16) { - seg = get_fs(); - if (!user_mode(regs)) - set_fs(KERNEL_DS); emulate_load_store_MIPS16e(regs, (void __user *)regs->cp0_badvaddr); - set_fs(seg); - return; - } + } goto sigbus; } if (unaligned_action == UNALIGNED_ACTION_SHOW) show_registers(regs); - pc = (unsigned int __user *)exception_epc(regs); + pc = (unsigned int *)exception_epc(regs); - seg = get_fs(); - if (!user_mode(regs)) - set_fs(KERNEL_DS); emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc); - set_fs(seg); return; sigbus: die_if_kernel("Kernel unaligned instruction access", regs); - force_sig(SIGBUS, current); + force_sig(SIGBUS); /* * XXX On return from the signal handler we should advance the epc @@ -1637,22 +1602,13 @@ sigbus: } #ifdef CONFIG_DEBUG_FS -extern struct dentry *mips_debugfs_dir; static int __init debugfs_unaligned(void) { - struct dentry *d; - - if (!mips_debugfs_dir) - return -ENODEV; - d = debugfs_create_u32("unaligned_instructions", S_IRUGO, - mips_debugfs_dir, &unaligned_instructions); - if (!d) - return -ENOMEM; - d = debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR, - mips_debugfs_dir, &unaligned_action); - if (!d) - return -ENOMEM; + debugfs_create_u32("unaligned_instructions", S_IRUGO, mips_debugfs_dir, + &unaligned_instructions); + debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR, + mips_debugfs_dir, &unaligned_action); return 0; } -__initcall(debugfs_unaligned); +arch_initcall(debugfs_unaligned); #endif diff --git a/arch/mips/kernel/uprobes.c b/arch/mips/kernel/uprobes.c new file mode 100644 index 000000000000..401b148f8917 --- /dev/null +++ b/arch/mips/kernel/uprobes.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/highmem.h> +#include <linux/kdebug.h> +#include <linux/types.h> +#include <linux/notifier.h> +#include <linux/sched.h> +#include <linux/uprobes.h> + +#include <asm/branch.h> +#include <asm/cpu-features.h> +#include <asm/ptrace.h> + +#include "probes-common.h" + +static inline int insn_has_delay_slot(const union mips_instruction insn) +{ + return __insn_has_delay_slot(insn); +} + +/** + * arch_uprobe_analyze_insn - instruction analysis including validity and fixups. + * @mm: the probed address space. + * @arch_uprobe: the probepoint information. + * @addr: virtual address at which to install the probepoint + * Return 0 on success or a -ve number on error. + */ +int arch_uprobe_analyze_insn(struct arch_uprobe *aup, + struct mm_struct *mm, unsigned long addr) +{ + union mips_instruction inst; + + /* + * For the time being this also blocks attempts to use uprobes with + * MIPS16 and microMIPS. + */ + if (addr & 0x03) + return -EINVAL; + + inst.word = aup->insn[0]; + + if (__insn_is_compact_branch(inst)) { + pr_notice("Uprobes for compact branches are not supported\n"); + return -EINVAL; + } + + aup->ixol[0] = aup->insn[insn_has_delay_slot(inst)]; + aup->ixol[1] = UPROBE_BRK_UPROBE_XOL; /* NOP */ + + return 0; +} + +/** + * is_trap_insn - check if the instruction is a trap variant + * @insn: instruction to be checked. + * Returns true if @insn is a trap variant. + * + * This definition overrides the weak definition in kernel/events/uprobes.c. + * and is needed for the case where an architecture has multiple trap + * instructions (like PowerPC or MIPS). We treat BREAK just like the more + * modern conditional trap instructions. + */ +bool is_trap_insn(uprobe_opcode_t *insn) +{ + union mips_instruction inst; + + inst.word = *insn; + + switch (inst.i_format.opcode) { + case spec_op: + switch (inst.r_format.func) { + case break_op: + case teq_op: + case tge_op: + case tgeu_op: + case tlt_op: + case tltu_op: + case tne_op: + return true; + } + break; + + case bcond_op: /* Yes, really ... */ + switch (inst.u_format.rt) { + case teqi_op: + case tgei_op: + case tgeiu_op: + case tlti_op: + case tltiu_op: + case tnei_op: + return true; + } + break; + } + + return false; +} + +#define UPROBE_TRAP_NR ULONG_MAX + +/* + * arch_uprobe_pre_xol - prepare to execute out of line. + * @auprobe: the probepoint information. + * @regs: reflects the saved user state of current task. + */ +int arch_uprobe_pre_xol(struct arch_uprobe *aup, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + /* + * Now find the EPC where to resume after the breakpoint has been + * dealt with. This may require emulation of a branch. + */ + aup->resume_epc = regs->cp0_epc + 4; + if (insn_has_delay_slot((union mips_instruction) aup->insn[0])) { + __compute_return_epc_for_insn(regs, + (union mips_instruction) aup->insn[0]); + aup->resume_epc = regs->cp0_epc; + } + utask->autask.saved_trap_nr = current->thread.trap_nr; + current->thread.trap_nr = UPROBE_TRAP_NR; + regs->cp0_epc = current->utask->xol_vaddr; + + return 0; +} + +int arch_uprobe_post_xol(struct arch_uprobe *aup, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + current->thread.trap_nr = utask->autask.saved_trap_nr; + regs->cp0_epc = aup->resume_epc; + + return 0; +} + +/* + * If xol insn itself traps and generates a signal(Say, + * SIGILL/SIGSEGV/etc), then detect the case where a singlestepped + * instruction jumps back to its own address. It is assumed that anything + * like do_page_fault/do_trap/etc sets thread.trap_nr != -1. + * + * arch_uprobe_pre_xol/arch_uprobe_post_xol save/restore thread.trap_nr, + * arch_uprobe_xol_was_trapped() simply checks that ->trap_nr is not equal to + * UPROBE_TRAP_NR == -1 set by arch_uprobe_pre_xol(). + */ +bool arch_uprobe_xol_was_trapped(struct task_struct *tsk) +{ + if (tsk->thread.trap_nr != UPROBE_TRAP_NR) + return true; + + return false; +} + +int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + struct die_args *args = data; + struct pt_regs *regs = args->regs; + + /* regs == NULL is a kernel bug */ + if (WARN_ON(!regs)) + return NOTIFY_DONE; + + /* We are only interested in userspace traps */ + if (!user_mode(regs)) + return NOTIFY_DONE; + + switch (val) { + case DIE_UPROBE: + if (uprobe_pre_sstep_notifier(regs)) + return NOTIFY_STOP; + break; + case DIE_UPROBE_XOL: + if (uprobe_post_sstep_notifier(regs)) + return NOTIFY_STOP; + break; + default: + break; + } + + return 0; +} + +/* + * This function gets called when XOL instruction either gets trapped or + * the thread has a fatal signal. Reset the instruction pointer to its + * probed address for the potential restart or for post mortem analysis. + */ +void arch_uprobe_abort_xol(struct arch_uprobe *aup, + struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + current->thread.trap_nr = utask->autask.saved_trap_nr; + instruction_pointer_set(regs, utask->vaddr); +} + +unsigned long arch_uretprobe_hijack_return_addr( + unsigned long trampoline_vaddr, struct pt_regs *regs) +{ + unsigned long ra; + + ra = regs->regs[31]; + + /* Replace the return address with the trampoline address */ + regs->regs[31] = trampoline_vaddr; + + return ra; +} + +void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, + void *src, unsigned long len) +{ + unsigned long kaddr, kstart; + + /* Initialize the slot */ + kaddr = (unsigned long)kmap_atomic(page); + kstart = kaddr + (vaddr & ~PAGE_MASK); + memcpy((void *)kstart, src, len); + flush_icache_range(kstart, kstart + len); + kunmap_atomic((void *)kaddr); +} + +/** + * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs + * @regs: Reflects the saved state of the task after it has hit a breakpoint + * instruction. + * Return the address of the breakpoint instruction. + * + * This overrides the weak version in kernel/events/uprobes.c. + */ +unsigned long uprobe_get_swbp_addr(struct pt_regs *regs) +{ + return instruction_pointer(regs); +} + +/* + * See if the instruction can be emulated. + * Returns true if instruction was emulated, false otherwise. + * + * For now we always emulate so this function just returns false. + */ +bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + return false; +} diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c index 0f1af58b036a..de096777172f 100644 --- a/arch/mips/kernel/vdso.c +++ b/arch/mips/kernel/vdso.c @@ -1,109 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2009, 2010 Cavium Networks, Inc. + * Copyright (C) 2015 Imagination Technologies + * Author: Alex Smith <alex.smith@imgtec.com> */ - -#include <linux/kernel.h> -#include <linux/err.h> -#include <linux/sched.h> -#include <linux/mm.h> -#include <linux/init.h> #include <linux/binfmts.h> #include <linux/elf.h> -#include <linux/vmalloc.h> -#include <linux/unistd.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/ioport.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/mman.h> +#include <linux/random.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/vdso_datastore.h> +#include <asm/abi.h> +#include <asm/mips-cps.h> +#include <asm/page.h> #include <asm/vdso.h> -#include <asm/uasm.h> +#include <vdso/helpers.h> +#include <vdso/vsyscall.h> -/* - * Including <asm/unistd.h> would give use the 64-bit syscall numbers ... - */ -#define __NR_O32_sigreturn 4119 -#define __NR_O32_rt_sigreturn 4193 -#define __NR_N32_rt_sigreturn 6211 +static_assert(VDSO_NR_PAGES == __VDSO_PAGES); -static struct page *vdso_page; - -static void __init install_trampoline(u32 *tramp, unsigned int sigreturn) +static void __init init_vdso_image(struct mips_vdso_image *image) { - uasm_i_addiu(&tramp, 2, 0, sigreturn); /* li v0, sigreturn */ - uasm_i_syscall(&tramp, 0); + unsigned long num_pages, i; + unsigned long data_pfn; + + BUG_ON(!PAGE_ALIGNED(image->data)); + BUG_ON(!PAGE_ALIGNED(image->size)); + + num_pages = image->size / PAGE_SIZE; + + data_pfn = __phys_to_pfn(__pa_symbol(image->data)); + for (i = 0; i < num_pages; i++) + image->mapping.pages[i] = pfn_to_page(data_pfn + i); } static int __init init_vdso(void) { - struct mips_vdso *vdso; - - vdso_page = alloc_page(GFP_KERNEL); - if (!vdso_page) - panic("Cannot allocate vdso"); - - vdso = vmap(&vdso_page, 1, 0, PAGE_KERNEL); - if (!vdso) - panic("Cannot map vdso"); - clear_page(vdso); - - install_trampoline(vdso->rt_signal_trampoline, __NR_rt_sigreturn); -#ifdef CONFIG_32BIT - install_trampoline(vdso->signal_trampoline, __NR_sigreturn); -#else - install_trampoline(vdso->n32_rt_signal_trampoline, - __NR_N32_rt_sigreturn); - install_trampoline(vdso->o32_signal_trampoline, __NR_O32_sigreturn); - install_trampoline(vdso->o32_rt_signal_trampoline, - __NR_O32_rt_sigreturn); + init_vdso_image(&vdso_image); + +#ifdef CONFIG_MIPS32_O32 + init_vdso_image(&vdso_image_o32); #endif - vunmap(vdso); +#ifdef CONFIG_MIPS32_N32 + init_vdso_image(&vdso_image_n32); +#endif return 0; } subsys_initcall(init_vdso); -static unsigned long vdso_addr(unsigned long start) +static unsigned long vdso_base(void) { - return STACK_TOP; + unsigned long base = STACK_TOP; + + if (IS_ENABLED(CONFIG_MIPS_FP_SUPPORT)) { + /* Skip the delay slot emulation page */ + base += PAGE_SIZE; + } + + if (current->flags & PF_RANDOMIZE) { + base += get_random_u32_below(VDSO_RANDOMIZE_SIZE); + base = PAGE_ALIGN(base); + } + + return base; } int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) { - int ret; - unsigned long addr; + struct mips_vdso_image *image = current->thread.abi->vdso; struct mm_struct *mm = current->mm; + unsigned long gic_size, size, base, data_addr, vdso_addr, gic_pfn, gic_base; + struct vm_area_struct *vma; + int ret; - down_write(&mm->mmap_sem); + if (mmap_write_lock_killable(mm)) + return -EINTR; - addr = vdso_addr(mm->start_stack); + if (IS_ENABLED(CONFIG_MIPS_FP_SUPPORT)) { + unsigned long unused; - addr = get_unmapped_area(NULL, addr, PAGE_SIZE, 0, 0); - if (IS_ERR_VALUE(addr)) { - ret = addr; - goto up_fail; + /* Map delay slot emulation page */ + base = do_mmap(NULL, STACK_TOP, PAGE_SIZE, PROT_READ | PROT_EXEC, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, 0, 0, &unused, + NULL); + if (IS_ERR_VALUE(base)) { + ret = base; + goto out; + } } - ret = install_special_mapping(mm, addr, PAGE_SIZE, - VM_READ|VM_EXEC| - VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, - &vdso_page); + /* + * Determine total area size. This includes the VDSO data itself, the + * data page, and the GIC user page if present. Always create a mapping + * for the GIC user area if the GIC is present regardless of whether it + * is the current clocksource, in case it comes into use later on. We + * only map a page even though the total area is 64K, as we only need + * the counter registers at the start. + */ + gic_size = mips_gic_present() ? PAGE_SIZE : 0; + size = gic_size + VDSO_NR_PAGES * PAGE_SIZE + image->size; + + /* + * Find a region that's large enough for us to perform the + * colour-matching alignment below. + */ + if (cpu_has_dc_aliases) + size += shm_align_mask + 1; + + base = get_unmapped_area(NULL, vdso_base(), size, 0, 0); + if (IS_ERR_VALUE(base)) { + ret = base; + goto out; + } - if (ret) - goto up_fail; + /* + * If we suffer from dcache aliasing, ensure that the VDSO data page + * mapping is coloured the same as the kernel's mapping of that memory. + * This ensures that when the kernel updates the VDSO data userland + * will observe it without requiring cache invalidations. + */ + if (cpu_has_dc_aliases) { + base = __ALIGN_MASK(base, shm_align_mask); + base += ((unsigned long)vdso_k_time_data - gic_size) & shm_align_mask; + } - mm->context.vdso = (void *)addr; + data_addr = base + gic_size; + vdso_addr = data_addr + VDSO_NR_PAGES * PAGE_SIZE; -up_fail: - up_write(&mm->mmap_sem); - return ret; -} + vma = vdso_install_vvar_mapping(mm, data_addr); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + goto out; + } -const char *arch_vma_name(struct vm_area_struct *vma) -{ - if (vma->vm_mm && vma->vm_start == (long)vma->vm_mm->context.vdso) - return "[vdso]"; - return NULL; + /* Map GIC user page. */ + if (gic_size) { + gic_base = (unsigned long)mips_gic_base + MIPS_GIC_USER_OFS; + gic_pfn = PFN_DOWN(__pa(gic_base)); + static const struct vm_special_mapping gic_mapping = { + .name = "[gic]", + .pages = (struct page **) { NULL }, + }; + + vma = _install_special_mapping(mm, base, gic_size, VM_READ | VM_MAYREAD, + &gic_mapping); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + goto out; + } + + ret = io_remap_pfn_range(vma, base, gic_pfn, gic_size, + pgprot_noncached(vma->vm_page_prot)); + if (ret) + goto out; + } + + /* Map VDSO image. */ + vma = _install_special_mapping(mm, vdso_addr, image->size, + VM_READ | VM_EXEC | + VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, + &image->mapping); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + goto out; + } + + mm->context.vdso = (void *)vdso_addr; + ret = 0; + +out: + mmap_write_unlock(mm); + return ret; } diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S index 05826d20a792..2b708fac8d2c 100644 --- a/arch/mips/kernel/vmlinux.lds.S +++ b/arch/mips/kernel/vmlinux.lds.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <asm/asm-offsets.h> #include <asm/thread_info.h> @@ -9,6 +10,13 @@ */ #define BSS_FIRST_SECTIONS *(.bss..swapper_pg_dir) +/* Cavium Octeon should not have a separate PT_NOTE Program Header. */ +#ifndef CONFIG_CAVIUM_OCTEON_SOC +#define EMITS_PT_NOTE +#endif + +#define RUNTIME_DISCARD_EXIT + #include <asm-generic/vmlinux.lds.h> #undef mips @@ -17,7 +25,9 @@ OUTPUT_ARCH(mips) ENTRY(kernel_entry) PHDRS { text PT_LOAD FLAGS(7); /* RWX */ +#ifndef CONFIG_CAVIUM_OCTEON_SOC note PT_NOTE FLAGS(4); /* R__ */ +#endif /* CAVIUM_OCTEON_SOC */ } #ifdef CONFIG_32BIT @@ -47,18 +57,21 @@ SECTIONS /* . = 0xa800000000300000; */ . = 0xffffffff80300000; #endif - . = VMLINUX_LOAD_ADDRESS; + . = LINKER_LOAD_ADDRESS; /* read-only */ _text = .; /* Text and read-only data */ .text : { + HEAD_TEXT TEXT_TEXT SCHED_TEXT LOCK_TEXT KPROBES_TEXT IRQENTRY_TEXT - *(.text.*) + SOFTIRQENTRY_TEXT *(.fixup) *(.gnu.warning) + . = ALIGN(16); + *(.got) /* Global offset table */ } :text = 0 _etext = .; /* End of text section */ @@ -67,15 +80,12 @@ SECTIONS /* Exception table for data bus errors */ __dbe_table : { __start___dbe_table = .; - *(__dbe_table) + KEEP(*(__dbe_table)) __stop___dbe_table = .; } - NOTES :text :note - .dummy : { *(.dummy) } :text - _sdata = .; /* Start of data section */ - RODATA + RO_DATA(4096) /* writeable */ .data : { /* Data */ @@ -83,11 +93,13 @@ SECTIONS INIT_TASK_DATA(THREAD_SIZE) NOSAVE_DATA + PAGE_ALIGNED_DATA(PAGE_SIZE) CACHELINE_ALIGNED_DATA(1 << CONFIG_MIPS_L1_CACHE_SHIFT) READ_MOSTLY_DATA(1 << CONFIG_MIPS_L1_CACHE_SHIFT) DATA_DATA CONSTRUCTORS } + BUG_TABLE _gp = . + 0x8000; .lit8 : { *(.lit8) @@ -112,7 +124,7 @@ SECTIONS . = ALIGN(4); .mips.machines.init : AT(ADDR(.mips.machines.init) - LOAD_OFFSET) { __mips_machines_start = .; - *(.mips.machines.init) + KEEP(*(.mips.machines.init)) __mips_machines_end = .; } @@ -125,8 +137,53 @@ SECTIONS .exit.data : { EXIT_DATA } - +#ifdef CONFIG_SMP PERCPU_SECTION(1 << CONFIG_MIPS_L1_CACHE_SHIFT) +#endif + + .rel.dyn : ALIGN(8) { + *(.rel) + *(.rel*) + } + +#ifdef CONFIG_MIPS_ELF_APPENDED_DTB + STRUCT_ALIGN(); + .appended_dtb : AT(ADDR(.appended_dtb) - LOAD_OFFSET) { + *(.appended_dtb) + KEEP(*(.appended_dtb)) + } +#endif + +#ifdef CONFIG_RELOCATABLE + . = ALIGN(4); + + .data.reloc : { + _relocation_start = .; + /* + * Space for relocation table + * This needs to be filled so that the + * relocs tool can overwrite the content. + * An invalid value is left at the start of the + * section to abort relocation if the table + * has not been filled in. + */ + LONG(0xFFFFFFFF); + FILL(0); + . += CONFIG_RELOCATION_TABLE_SIZE - 4; + _relocation_end = .; + } +#endif + +#ifdef CONFIG_MIPS_RAW_APPENDED_DTB + .fill : { + FILL(0); + BYTE(0); + STRUCT_ALIGN(); + } + __appended_dtb = .; + /* leave space for appended DTB */ + . += 0x100000; +#endif /* * Align to 64K in attempt to eliminate holes before the * .bss..swapper_pg_dir section at the start of .bss. This @@ -141,7 +198,7 @@ SECTIONS * Force .bss to 64K alignment so that .bss..swapper_pg_dir * gets that alignment. .sbss should be empty, so there will be * no holes after __init_end. */ - BSS_SECTION(0, 0x10000, 0) + BSS_SECTION(0, 0x10000, 8) _end = . ; @@ -160,6 +217,7 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + ELF_DETAILS /* These must appear regardless of . */ .gptab.sdata : { @@ -175,7 +233,9 @@ SECTIONS DISCARDS /DISCARD/ : { /* ABI crap starts here */ + *(.MIPS.abiflags) *(.MIPS.options) + *(.gnu.attributes) *(.options) *(.pdr) *(.reginfo) diff --git a/arch/mips/kernel/vpe-mt.c b/arch/mips/kernel/vpe-mt.c new file mode 100644 index 000000000000..84124ac2d2a5 --- /dev/null +++ b/arch/mips/kernel/vpe-mt.c @@ -0,0 +1,519 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2004, 2005 MIPS Technologies, Inc. All rights reserved. + * Copyright (C) 2013 Imagination Technologies Ltd. + */ +#include <linux/kernel.h> +#include <linux/device.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/export.h> + +#include <asm/mipsregs.h> +#include <asm/mipsmtregs.h> +#include <asm/mips_mt.h> +#include <asm/vpe.h> + +static int major; + +/* The number of TCs and VPEs physically available on the core */ +static int hw_tcs, hw_vpes; + +/* We are prepared so configure and start the VPE... */ +int vpe_run(struct vpe *v) +{ + unsigned long flags, val, dmt_flag; + struct vpe_notifications *notifier; + unsigned int vpeflags; + struct tc *t; + + /* check we are the Master VPE */ + local_irq_save(flags); + val = read_c0_vpeconf0(); + if (!(val & VPECONF0_MVP)) { + pr_warn("VPE loader: only Master VPE's are able to config MT\n"); + local_irq_restore(flags); + + return -1; + } + + dmt_flag = dmt(); + vpeflags = dvpe(); + + if (list_empty(&v->tc)) { + evpe(vpeflags); + emt(dmt_flag); + local_irq_restore(flags); + + pr_warn("VPE loader: No TC's associated with VPE %d\n", + v->minor); + + return -ENOEXEC; + } + + t = list_first_entry(&v->tc, struct tc, tc); + + /* Put MVPE's into 'configuration state' */ + set_c0_mvpcontrol(MVPCONTROL_VPC); + + settc(t->index); + + /* should check it is halted, and not activated */ + if ((read_tc_c0_tcstatus() & TCSTATUS_A) || + !(read_tc_c0_tchalt() & TCHALT_H)) { + evpe(vpeflags); + emt(dmt_flag); + local_irq_restore(flags); + + pr_warn("VPE loader: TC %d is already active!\n", + t->index); + + return -ENOEXEC; + } + + /* + * Write the address we want it to start running from in the TCPC + * register. + */ + write_tc_c0_tcrestart((unsigned long)v->__start); + write_tc_c0_tccontext((unsigned long)0); + + /* + * Mark the TC as activated, not interrupt exempt and not dynamically + * allocatable + */ + val = read_tc_c0_tcstatus(); + val = (val & ~(TCSTATUS_DA | TCSTATUS_IXMT)) | TCSTATUS_A; + write_tc_c0_tcstatus(val); + + write_tc_c0_tchalt(read_tc_c0_tchalt() & ~TCHALT_H); + + /* + * We don't pass the memsize here, so VPE programs need to be + * compiled with DFLT_STACK_SIZE and DFLT_HEAP_SIZE defined. + */ + mttgpr($7, 0); + mttgpr($6, v->ntcs); + + /* set up VPE1 */ + /* + * bind the TC to VPE 1 as late as possible so we only have the final + * VPE registers to set up, and so an EJTAG probe can trigger on it + */ + write_tc_c0_tcbind((read_tc_c0_tcbind() & ~TCBIND_CURVPE) | 1); + + write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() & ~(VPECONF0_VPA)); + + back_to_back_c0_hazard(); + + /* Set up the XTC bit in vpeconf0 to point at our tc */ + write_vpe_c0_vpeconf0((read_vpe_c0_vpeconf0() & ~(VPECONF0_XTC)) + | (t->index << VPECONF0_XTC_SHIFT)); + + back_to_back_c0_hazard(); + + /* enable this VPE */ + write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA); + + /* clear out any left overs from a previous program */ + write_vpe_c0_status(0); + write_vpe_c0_cause(0); + + /* take system out of configuration state */ + clear_c0_mvpcontrol(MVPCONTROL_VPC); + + /* + * SMVP kernels manage VPE enable independently, but uniprocessor + * kernels need to turn it on, even if that wasn't the pre-dvpe() state. + */ +#ifdef CONFIG_SMP + evpe(vpeflags); +#else + evpe(EVPE_ENABLE); +#endif + emt(dmt_flag); + local_irq_restore(flags); + + list_for_each_entry(notifier, &v->notify, list) + notifier->start(VPE_MODULE_MINOR); + + return 0; +} + +void cleanup_tc(struct tc *tc) +{ + unsigned long flags; + unsigned int mtflags, vpflags; + int tmp; + + local_irq_save(flags); + mtflags = dmt(); + vpflags = dvpe(); + /* Put MVPE's into 'configuration state' */ + set_c0_mvpcontrol(MVPCONTROL_VPC); + + settc(tc->index); + tmp = read_tc_c0_tcstatus(); + + /* mark not allocated and not dynamically allocatable */ + tmp &= ~(TCSTATUS_A | TCSTATUS_DA); + tmp |= TCSTATUS_IXMT; /* interrupt exempt */ + write_tc_c0_tcstatus(tmp); + + write_tc_c0_tchalt(TCHALT_H); + mips_ihb(); + + clear_c0_mvpcontrol(MVPCONTROL_VPC); + evpe(vpflags); + emt(mtflags); + local_irq_restore(flags); +} + +/* module wrapper entry points */ +/* give me a vpe */ +void *vpe_alloc(void) +{ + int i; + struct vpe *v; + + /* find a vpe */ + for (i = 1; i < MAX_VPES; i++) { + v = get_vpe(i); + if (v != NULL) { + v->state = VPE_STATE_INUSE; + return v; + } + } + return NULL; +} +EXPORT_SYMBOL(vpe_alloc); + +/* start running from here */ +int vpe_start(void *vpe, unsigned long start) +{ + struct vpe *v = vpe; + + v->__start = start; + return vpe_run(v); +} +EXPORT_SYMBOL(vpe_start); + +/* halt it for now */ +int vpe_stop(void *vpe) +{ + struct vpe *v = vpe; + struct tc *t; + unsigned int evpe_flags; + + evpe_flags = dvpe(); + + t = list_entry(v->tc.next, struct tc, tc); + if (t != NULL) { + settc(t->index); + write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() & ~VPECONF0_VPA); + } + + evpe(evpe_flags); + + return 0; +} +EXPORT_SYMBOL(vpe_stop); + +/* I've done with it thank you */ +int vpe_free(void *vpe) +{ + struct vpe *v = vpe; + struct tc *t; + unsigned int evpe_flags; + + t = list_entry(v->tc.next, struct tc, tc); + if (t == NULL) + return -ENOEXEC; + + evpe_flags = dvpe(); + + /* Put MVPE's into 'configuration state' */ + set_c0_mvpcontrol(MVPCONTROL_VPC); + + settc(t->index); + write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() & ~VPECONF0_VPA); + + /* halt the TC */ + write_tc_c0_tchalt(TCHALT_H); + mips_ihb(); + + /* mark the TC unallocated */ + write_tc_c0_tcstatus(read_tc_c0_tcstatus() & ~TCSTATUS_A); + + v->state = VPE_STATE_UNUSED; + + clear_c0_mvpcontrol(MVPCONTROL_VPC); + evpe(evpe_flags); + + return 0; +} +EXPORT_SYMBOL(vpe_free); + +static ssize_t store_kill(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct vpe *vpe = get_vpe(aprp_cpu_index()); + struct vpe_notifications *notifier; + + list_for_each_entry(notifier, &vpe->notify, list) + notifier->stop(aprp_cpu_index()); + + release_progmem(vpe->load_addr); + cleanup_tc(get_tc(aprp_cpu_index())); + vpe_stop(vpe); + vpe_free(vpe); + + return len; +} +static DEVICE_ATTR(kill, S_IWUSR, NULL, store_kill); + +static ssize_t ntcs_show(struct device *cd, struct device_attribute *attr, + char *buf) +{ + struct vpe *vpe = get_vpe(aprp_cpu_index()); + + return sprintf(buf, "%d\n", vpe->ntcs); +} + +static ssize_t ntcs_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct vpe *vpe = get_vpe(aprp_cpu_index()); + unsigned long new; + int ret; + + ret = kstrtoul(buf, 0, &new); + if (ret < 0) + return ret; + + if (new == 0 || new > (hw_tcs - aprp_cpu_index())) + return -EINVAL; + + vpe->ntcs = new; + + return len; +} +static DEVICE_ATTR_RW(ntcs); + +static struct attribute *vpe_attrs[] = { + &dev_attr_kill.attr, + &dev_attr_ntcs.attr, + NULL, +}; +ATTRIBUTE_GROUPS(vpe); + +static void vpe_device_release(struct device *cd) +{ +} + +static struct class vpe_class = { + .name = "vpe", + .dev_release = vpe_device_release, + .dev_groups = vpe_groups, +}; + +static struct device vpe_device; + +int __init vpe_module_init(void) +{ + unsigned int mtflags, vpflags; + unsigned long flags, val; + struct vpe *v = NULL; + struct tc *t; + int tc, err; + + if (!cpu_has_mipsmt) { + pr_warn("VPE loader: not a MIPS MT capable processor\n"); + return -ENODEV; + } + + if (vpelimit == 0) { + pr_warn("No VPEs reserved for AP/SP, not initialize VPE loader\n" + "Pass maxvpes=<n> argument as kernel argument\n"); + + return -ENODEV; + } + + if (aprp_cpu_index() == 0) { + pr_warn("No TCs reserved for AP/SP, not initialize VPE loader\n" + "Pass maxtcs=<n> argument as kernel argument\n"); + + return -ENODEV; + } + + major = register_chrdev(0, VPE_MODULE_NAME, &vpe_fops); + if (major < 0) { + pr_warn("VPE loader: unable to register character device\n"); + return major; + } + + err = class_register(&vpe_class); + if (err) { + pr_err("vpe_class registration failed\n"); + goto out_chrdev; + } + + device_initialize(&vpe_device); + vpe_device.class = &vpe_class; + vpe_device.parent = NULL; + dev_set_name(&vpe_device, "vpe1"); + vpe_device.devt = MKDEV(major, VPE_MODULE_MINOR); + err = device_add(&vpe_device); + if (err) { + pr_err("Adding vpe_device failed\n"); + goto out_class; + } + + local_irq_save(flags); + mtflags = dmt(); + vpflags = dvpe(); + + /* Put MVPE's into 'configuration state' */ + set_c0_mvpcontrol(MVPCONTROL_VPC); + + val = read_c0_mvpconf0(); + hw_tcs = (val & MVPCONF0_PTC) + 1; + hw_vpes = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; + + for (tc = aprp_cpu_index(); tc < hw_tcs; tc++) { + /* + * Must re-enable multithreading temporarily or in case we + * reschedule send IPIs or similar we might hang. + */ + clear_c0_mvpcontrol(MVPCONTROL_VPC); + evpe(vpflags); + emt(mtflags); + local_irq_restore(flags); + t = alloc_tc(tc); + if (!t) { + err = -ENOMEM; + goto out_dev; + } + + local_irq_save(flags); + mtflags = dmt(); + vpflags = dvpe(); + set_c0_mvpcontrol(MVPCONTROL_VPC); + + /* VPE's */ + if (tc < hw_tcs) { + settc(tc); + + v = alloc_vpe(tc); + if (v == NULL) { + pr_warn("VPE: unable to allocate VPE\n"); + goto out_reenable; + } + + v->ntcs = hw_tcs - aprp_cpu_index(); + + /* add the tc to the list of this vpe's tc's. */ + list_add(&t->tc, &v->tc); + + /* deactivate all but vpe0 */ + if (tc >= aprp_cpu_index()) { + unsigned long tmp = read_vpe_c0_vpeconf0(); + + tmp &= ~VPECONF0_VPA; + + /* master VPE */ + tmp |= VPECONF0_MVP; + write_vpe_c0_vpeconf0(tmp); + } + + /* disable multi-threading with TC's */ + write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() & + ~VPECONTROL_TE); + + if (tc >= vpelimit) { + /* + * Set config to be the same as vpe0, + * particularly kseg0 coherency alg + */ + write_vpe_c0_config(read_c0_config()); + } + } + + /* TC's */ + t->pvpe = v; /* set the parent vpe */ + + if (tc >= aprp_cpu_index()) { + unsigned long tmp; + + settc(tc); + + /* + * A TC that is bound to any other VPE gets bound to + * VPE0, ideally I'd like to make it homeless but it + * doesn't appear to let me bind a TC to a non-existent + * VPE. Which is perfectly reasonable. + * + * The (un)bound state is visible to an EJTAG probe so + * may notify GDB... + */ + tmp = read_tc_c0_tcbind(); + if (tmp & TCBIND_CURVPE) { + /* tc is bound >vpe0 */ + write_tc_c0_tcbind(tmp & ~TCBIND_CURVPE); + + t->pvpe = get_vpe(0); /* set the parent vpe */ + } + + /* halt the TC */ + write_tc_c0_tchalt(TCHALT_H); + mips_ihb(); + + tmp = read_tc_c0_tcstatus(); + + /* mark not activated and not dynamically allocatable */ + tmp &= ~(TCSTATUS_A | TCSTATUS_DA); + tmp |= TCSTATUS_IXMT; /* interrupt exempt */ + write_tc_c0_tcstatus(tmp); + } + } + +out_reenable: + /* release config state */ + clear_c0_mvpcontrol(MVPCONTROL_VPC); + + evpe(vpflags); + emt(mtflags); + local_irq_restore(flags); + + return 0; + +out_dev: + device_del(&vpe_device); + +out_class: + put_device(&vpe_device); + class_unregister(&vpe_class); + +out_chrdev: + unregister_chrdev(major, VPE_MODULE_NAME); + + return err; +} + +void __exit vpe_module_exit(void) +{ + struct vpe *v, *n; + + device_unregister(&vpe_device); + class_unregister(&vpe_class); + unregister_chrdev(major, VPE_MODULE_NAME); + + /* No locking needed here */ + list_for_each_entry_safe(v, n, &vpecontrol.vpe_list, list) { + if (v->state != VPE_STATE_UNUSED) + release_vpe(v); + } +} diff --git a/arch/mips/kernel/vpe.c b/arch/mips/kernel/vpe.c index 1765bab000a0..2b67c44adab9 100644 --- a/arch/mips/kernel/vpe.c +++ b/arch/mips/kernel/vpe.c @@ -1,58 +1,41 @@ /* - * Copyright (C) 2004, 2005 MIPS Technologies, Inc. All rights reserved. - * - * This program is free software; you can distribute it and/or modify it - * under the terms of the GNU General Public License (Version 2) as - * published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. - */ - -/* - * VPE support module - * - * Provides support for loading a MIPS SP program on VPE1. - * The SP environment is rather simple, no tlb's. It needs to be relocatable - * (or partially linked). You should initialise your stack in the startup - * code. This loader looks for the symbol __start and sets up - * execution to resume from there. The MIPS SDE kit contains suitable examples. + * Copyright (C) 2004, 2005 MIPS Technologies, Inc. All rights reserved. + * Copyright (C) 2013 Imagination Technologies Ltd. * - * To load and run, simply cat a SP 'program file' to /dev/vpe1. - * i.e cat spapp >/dev/vpe1. + * VPE support module for loading a MIPS SP program into VPE1. The SP + * environment is rather simple since there are no TLBs. It needs + * to be relocatable (or partially linked). Initialize your stack in + * the startup-code. The loader looks for the symbol __start and sets + * up the execution to resume from there. To load and run, simply do + * a cat SP 'binary' to the /dev/vpe1 device. */ #include <linux/kernel.h> #include <linux/device.h> #include <linux/fs.h> #include <linux/init.h> -#include <asm/uaccess.h> #include <linux/slab.h> #include <linux/list.h> #include <linux/vmalloc.h> #include <linux/elf.h> #include <linux/seq_file.h> +#include <linux/string.h> #include <linux/syscalls.h> #include <linux/moduleloader.h> #include <linux/interrupt.h> #include <linux/poll.h> -#include <linux/bootmem.h> +#include <linux/memblock.h> #include <asm/mipsregs.h> #include <asm/mipsmtregs.h> #include <asm/cacheflush.h> #include <linux/atomic.h> -#include <asm/cpu.h> #include <asm/mips_mt.h> #include <asm/processor.h> #include <asm/vpe.h> -typedef void *vpe_handle; - #ifndef ARCH_SHF_SMALL #define ARCH_SHF_SMALL 0 #endif @@ -60,96 +43,15 @@ typedef void *vpe_handle; /* If this is set, the section belongs in the init part of the module */ #define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG-1)) -/* - * The number of TCs and VPEs physically available on the core - */ -static int hw_tcs, hw_vpes; -static char module_name[] = "vpe"; -static int major; -static const int minor = 1; /* fixed for now */ - -/* grab the likely amount of memory we will need. */ -#ifdef CONFIG_MIPS_VPE_LOADER_TOM -#define P_SIZE (2 * 1024 * 1024) -#else -/* add an overhead to the max kmalloc size for non-striped symbols/etc */ -#define P_SIZE (256 * 1024) -#endif - -extern unsigned long physical_memsize; - -#define MAX_VPES 16 -#define VPE_PATH_MAX 256 - -enum vpe_state { - VPE_STATE_UNUSED = 0, - VPE_STATE_INUSE, - VPE_STATE_RUNNING -}; - -enum tc_state { - TC_STATE_UNUSED = 0, - TC_STATE_INUSE, - TC_STATE_RUNNING, - TC_STATE_DYNAMIC -}; - -struct vpe { - enum vpe_state state; - - /* (device) minor associated with this vpe */ - int minor; - - /* elfloader stuff */ - void *load_addr; - unsigned long len; - char *pbuffer; - unsigned long plen; - unsigned int uid, gid; - char cwd[VPE_PATH_MAX]; - - unsigned long __start; - - /* tc's associated with this vpe */ - struct list_head tc; - - /* The list of vpe's */ - struct list_head list; - - /* shared symbol address */ - void *shared_ptr; - - /* the list of who wants to know when something major happens */ - struct list_head notify; - - unsigned int ntcs; -}; - -struct tc { - enum tc_state state; - int index; - - struct vpe *pvpe; /* parent VPE */ - struct list_head tc; /* The list of TC's with this VPE */ - struct list_head list; /* The global list of tc's */ -}; - -struct { - spinlock_t vpe_list_lock; - struct list_head vpe_list; /* Virtual processing elements */ - spinlock_t tc_list_lock; - struct list_head tc_list; /* Thread contexts */ -} vpecontrol = { +struct vpe_control vpecontrol = { .vpe_list_lock = __SPIN_LOCK_UNLOCKED(vpe_list_lock), .vpe_list = LIST_HEAD_INIT(vpecontrol.vpe_list), .tc_list_lock = __SPIN_LOCK_UNLOCKED(tc_list_lock), .tc_list = LIST_HEAD_INIT(vpecontrol.tc_list) }; -static void release_progmem(void *ptr); - /* get the vpe associated with this minor */ -static struct vpe *get_vpe(int minor) +struct vpe *get_vpe(int minor) { struct vpe *res, *v; @@ -159,7 +61,7 @@ static struct vpe *get_vpe(int minor) res = NULL; spin_lock(&vpecontrol.vpe_list_lock); list_for_each_entry(v, &vpecontrol.vpe_list, list) { - if (v->minor == minor) { + if (v->minor == VPE_MODULE_MINOR) { res = v; break; } @@ -170,7 +72,7 @@ static struct vpe *get_vpe(int minor) } /* get the vpe associated with this minor */ -static struct tc *get_tc(int index) +struct tc *get_tc(int index) { struct tc *res, *t; @@ -188,12 +90,13 @@ static struct tc *get_tc(int index) } /* allocate a vpe and associate it with this minor (or index) */ -static struct vpe *alloc_vpe(int minor) +struct vpe *alloc_vpe(int minor) { struct vpe *v; - if ((v = kzalloc(sizeof(struct vpe), GFP_KERNEL)) == NULL) - return NULL; + v = kzalloc(sizeof(struct vpe), GFP_KERNEL); + if (v == NULL) + goto out; INIT_LIST_HEAD(&v->tc); spin_lock(&vpecontrol.vpe_list_lock); @@ -201,17 +104,19 @@ static struct vpe *alloc_vpe(int minor) spin_unlock(&vpecontrol.vpe_list_lock); INIT_LIST_HEAD(&v->notify); - v->minor = minor; + v->minor = VPE_MODULE_MINOR; +out: return v; } /* allocate a tc. At startup only tc0 is running, all other can be halted. */ -static struct tc *alloc_tc(int index) +struct tc *alloc_tc(int index) { struct tc *tc; - if ((tc = kzalloc(sizeof(struct tc), GFP_KERNEL)) == NULL) + tc = kzalloc(sizeof(struct tc), GFP_KERNEL); + if (tc == NULL) goto out; INIT_LIST_HEAD(&tc->tc); @@ -226,36 +131,16 @@ out: } /* clean up and free everything */ -static void release_vpe(struct vpe *v) +void release_vpe(struct vpe *v) { list_del(&v->list); if (v->load_addr) - release_progmem(v); + release_progmem(v->load_addr); kfree(v); } -static void __maybe_unused dump_mtregs(void) -{ - unsigned long val; - - val = read_c0_config3(); - printk("config3 0x%lx MT %ld\n", val, - (val & CONFIG3_MT) >> CONFIG3_MT_SHIFT); - - val = read_c0_mvpcontrol(); - printk("MVPControl 0x%lx, STLB %ld VPC %ld EVP %ld\n", val, - (val & MVPCONTROL_STLB) >> MVPCONTROL_STLB_SHIFT, - (val & MVPCONTROL_VPC) >> MVPCONTROL_VPC_SHIFT, - (val & MVPCONTROL_EVP)); - - val = read_c0_mvpconf0(); - printk("mvpconf0 0x%lx, PVPE %ld PTC %ld M %ld\n", val, - (val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT, - val & MVPCONF0_PTC, (val & MVPCONF0_M) >> MVPCONF0_M_SHIFT); -} - -/* Find some VPE program space */ -static void *alloc_progmem(unsigned long len) +/* Find some VPE program space */ +void *alloc_progmem(unsigned long len) { void *addr; @@ -274,7 +159,7 @@ static void *alloc_progmem(unsigned long len) return addr; } -static void release_progmem(void *ptr) +void release_progmem(void *ptr) { #ifndef CONFIG_MIPS_VPE_LOADER_TOM kfree(ptr); @@ -282,7 +167,7 @@ static void release_progmem(void *ptr) } /* Update size with this section: return offset. */ -static long get_offset(unsigned long *size, Elf_Shdr * sechdr) +static long get_offset(unsigned long *size, Elf_Shdr *sechdr) { long ret; @@ -295,8 +180,8 @@ static long get_offset(unsigned long *size, Elf_Shdr * sechdr) might -- code, read-only data, read-write data, small data. Tally sizes, and place the offsets into sh_entsize fields: high bit means it belongs in init. */ -static void layout_sections(struct module *mod, const Elf_Ehdr * hdr, - Elf_Shdr * sechdrs, const char *secstrings) +static void layout_sections(struct module *mod, const Elf_Ehdr *hdr, + Elf_Shdr *sechdrs, const char *secstrings) { static unsigned long const masks[][2] = { /* NOTE: all executable code must be the first section @@ -315,23 +200,20 @@ static void layout_sections(struct module *mod, const Elf_Ehdr * hdr, for (m = 0; m < ARRAY_SIZE(masks); ++m) { for (i = 0; i < hdr->e_shnum; ++i) { Elf_Shdr *s = &sechdrs[i]; + struct module_memory *mod_mem; + + mod_mem = &mod->mem[MOD_TEXT]; - // || strncmp(secstrings + s->sh_name, ".init", 5) == 0) if ((s->sh_flags & masks[m][0]) != masks[m][0] || (s->sh_flags & masks[m][1]) || s->sh_entsize != ~0UL) continue; s->sh_entsize = - get_offset((unsigned long *)&mod->core_size, s); + get_offset((unsigned long *)&mod_mem->size, s); } - - if (m == 0) - mod->core_text_size = mod->core_size; - } } - /* from module-elf32.c, but subverted a little */ struct mips_hi16 { @@ -354,20 +236,18 @@ static int apply_r_mips_gprel16(struct module *me, uint32_t *location, { int rel; - if( !(*location & 0xffff) ) { + if (!(*location & 0xffff)) { rel = (int)v - gp_addr; - } - else { + } else { /* .sbss + gp(relative) + offset */ /* kludge! */ rel = (int)(short)((int)v + gp_offs + (int)(short)(*location & 0xffff) - gp_addr); } - if( (rel > 32768) || (rel < -32768) ) { - printk(KERN_DEBUG "VPE loader: apply_r_mips_gprel16: " - "relative address 0x%x out of range of gp register\n", - rel); + if ((rel > 32768) || (rel < -32768)) { + pr_debug("VPE loader: apply_r_mips_gprel16: relative address 0x%x out of range of gp register\n", + rel); return -ENOEXEC; } @@ -381,12 +261,12 @@ static int apply_r_mips_pc16(struct module *me, uint32_t *location, { int rel; rel = (((unsigned int)v - (unsigned int)location)); - rel >>= 2; // because the offset is in _instructions_ not bytes. - rel -= 1; // and one instruction less due to the branch delay slot. + rel >>= 2; /* because the offset is in _instructions_ not bytes. */ + rel -= 1; /* and one instruction less due to the branch delay slot. */ - if( (rel > 32768) || (rel < -32768) ) { - printk(KERN_DEBUG "VPE loader: " - "apply_r_mips_pc16: relative address out of range 0x%x\n", rel); + if ((rel > 32768) || (rel < -32768)) { + pr_debug("VPE loader: apply_r_mips_pc16: relative address out of range 0x%x\n", + rel); return -ENOEXEC; } @@ -407,8 +287,7 @@ static int apply_r_mips_26(struct module *me, uint32_t *location, Elf32_Addr v) { if (v % 4) { - printk(KERN_DEBUG "VPE loader: apply_r_mips_26 " - " unaligned relocation\n"); + pr_debug("VPE loader: apply_r_mips_26: unaligned relocation\n"); return -ENOEXEC; } @@ -439,7 +318,7 @@ static int apply_r_mips_hi16(struct module *me, uint32_t *location, * the carry we need to add. Save the information, and let LO16 do the * actual relocation. */ - n = kmalloc(sizeof *n, GFP_KERNEL); + n = kmalloc(sizeof(*n), GFP_KERNEL); if (!n) return -ENOMEM; @@ -471,9 +350,7 @@ static int apply_r_mips_lo16(struct module *me, uint32_t *location, * The value for the HI16 had best be the same. */ if (v != l->value) { - printk(KERN_DEBUG "VPE loader: " - "apply_r_mips_lo16/hi16: \t" - "inconsistent value information\n"); + pr_debug("VPE loader: apply_r_mips_lo16/hi16: inconsistent value information\n"); goto out_free; } @@ -569,20 +446,19 @@ static int apply_relocations(Elf32_Shdr *sechdrs, + ELF32_R_SYM(r_info); if (!sym->st_value) { - printk(KERN_DEBUG "%s: undefined weak symbol %s\n", - me->name, strtab + sym->st_name); + pr_debug("%s: undefined weak symbol %s\n", + me->name, strtab + sym->st_name); /* just print the warning, dont barf */ } v = sym->st_value; res = reloc_handlers[ELF32_R_TYPE(r_info)](me, location, v); - if( res ) { + if (res) { char *r = rstrs[ELF32_R_TYPE(r_info)]; - printk(KERN_WARNING "VPE loader: .text+0x%x " - "relocation type %s for symbol \"%s\" failed\n", - rel[i].r_offset, r ? r : "UNKNOWN", - strtab + sym->st_name); + pr_warn("VPE loader: .text+0x%x relocation type %s for symbol \"%s\" failed\n", + rel[i].r_offset, r ? r : "UNKNOWN", + strtab + sym->st_name); return res; } } @@ -597,10 +473,8 @@ static inline void save_gp_address(unsigned int secbase, unsigned int rel) } /* end module-elf32.c */ - - /* Change all symbols so that sh_value encodes the pointer directly. */ -static void simplify_symbols(Elf_Shdr * sechdrs, +static void simplify_symbols(Elf_Shdr *sechdrs, unsigned int symindex, const char *strtab, const char *secstrings, @@ -641,18 +515,16 @@ static void simplify_symbols(Elf_Shdr * sechdrs, break; case SHN_MIPS_SCOMMON: - printk(KERN_DEBUG "simplify_symbols: ignoring SHN_MIPS_SCOMMON " - "symbol <%s> st_shndx %d\n", strtab + sym[i].st_name, - sym[i].st_shndx); - // .sbss section + pr_debug("simplify_symbols: ignoring SHN_MIPS_SCOMMON symbol <%s> st_shndx %d\n", + strtab + sym[i].st_name, sym[i].st_shndx); + /* .sbss section */ break; default: secbase = sechdrs[sym[i].st_shndx].sh_addr; - if (strncmp(strtab + sym[i].st_name, "_gp", 3) == 0) { + if (strncmp(strtab + sym[i].st_name, "_gp", 3) == 0) save_gp_address(secbase, sym[i].st_value); - } sym[i].st_value += secbase; break; @@ -661,142 +533,21 @@ static void simplify_symbols(Elf_Shdr * sechdrs, } #ifdef DEBUG_ELFLOADER -static void dump_elfsymbols(Elf_Shdr * sechdrs, unsigned int symindex, +static void dump_elfsymbols(Elf_Shdr *sechdrs, unsigned int symindex, const char *strtab, struct module *mod) { Elf_Sym *sym = (void *)sechdrs[symindex].sh_addr; unsigned int i, n = sechdrs[symindex].sh_size / sizeof(Elf_Sym); - printk(KERN_DEBUG "dump_elfsymbols: n %d\n", n); + pr_debug("dump_elfsymbols: n %d\n", n); for (i = 1; i < n; i++) { - printk(KERN_DEBUG " i %d name <%s> 0x%x\n", i, - strtab + sym[i].st_name, sym[i].st_value); + pr_debug(" i %d name <%s> 0x%x\n", i, strtab + sym[i].st_name, + sym[i].st_value); } } #endif -/* We are prepared so configure and start the VPE... */ -static int vpe_run(struct vpe * v) -{ - unsigned long flags, val, dmt_flag; - struct vpe_notifications *n; - unsigned int vpeflags; - struct tc *t; - - /* check we are the Master VPE */ - local_irq_save(flags); - val = read_c0_vpeconf0(); - if (!(val & VPECONF0_MVP)) { - printk(KERN_WARNING - "VPE loader: only Master VPE's are allowed to configure MT\n"); - local_irq_restore(flags); - - return -1; - } - - dmt_flag = dmt(); - vpeflags = dvpe(); - - if (list_empty(&v->tc)) { - evpe(vpeflags); - emt(dmt_flag); - local_irq_restore(flags); - - printk(KERN_WARNING - "VPE loader: No TC's associated with VPE %d\n", - v->minor); - - return -ENOEXEC; - } - - t = list_first_entry(&v->tc, struct tc, tc); - - /* Put MVPE's into 'configuration state' */ - set_c0_mvpcontrol(MVPCONTROL_VPC); - - settc(t->index); - - /* should check it is halted, and not activated */ - if ((read_tc_c0_tcstatus() & TCSTATUS_A) || !(read_tc_c0_tchalt() & TCHALT_H)) { - evpe(vpeflags); - emt(dmt_flag); - local_irq_restore(flags); - - printk(KERN_WARNING "VPE loader: TC %d is already active!\n", - t->index); - - return -ENOEXEC; - } - - /* Write the address we want it to start running from in the TCPC register. */ - write_tc_c0_tcrestart((unsigned long)v->__start); - write_tc_c0_tccontext((unsigned long)0); - - /* - * Mark the TC as activated, not interrupt exempt and not dynamically - * allocatable - */ - val = read_tc_c0_tcstatus(); - val = (val & ~(TCSTATUS_DA | TCSTATUS_IXMT)) | TCSTATUS_A; - write_tc_c0_tcstatus(val); - - write_tc_c0_tchalt(read_tc_c0_tchalt() & ~TCHALT_H); - - /* - * The sde-kit passes 'memsize' to __start in $a3, so set something - * here... Or set $a3 to zero and define DFLT_STACK_SIZE and - * DFLT_HEAP_SIZE when you compile your program - */ - mttgpr(6, v->ntcs); - mttgpr(7, physical_memsize); - - /* set up VPE1 */ - /* - * bind the TC to VPE 1 as late as possible so we only have the final - * VPE registers to set up, and so an EJTAG probe can trigger on it - */ - write_tc_c0_tcbind((read_tc_c0_tcbind() & ~TCBIND_CURVPE) | 1); - - write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() & ~(VPECONF0_VPA)); - - back_to_back_c0_hazard(); - - /* Set up the XTC bit in vpeconf0 to point at our tc */ - write_vpe_c0_vpeconf0( (read_vpe_c0_vpeconf0() & ~(VPECONF0_XTC)) - | (t->index << VPECONF0_XTC_SHIFT)); - - back_to_back_c0_hazard(); - - /* enable this VPE */ - write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA); - - /* clear out any left overs from a previous program */ - write_vpe_c0_status(0); - write_vpe_c0_cause(0); - - /* take system out of configuration state */ - clear_c0_mvpcontrol(MVPCONTROL_VPC); - - /* - * SMTC/SMVP kernels manage VPE enable independently, - * but uniprocessor kernels need to turn it on, even - * if that wasn't the pre-dvpe() state. - */ -#ifdef CONFIG_SMP - evpe(vpeflags); -#else - evpe(EVPE_ENABLE); -#endif - emt(dmt_flag); - local_irq_restore(flags); - - list_for_each_entry(n, &v->notify, list) - n->start(minor); - - return 0; -} - -static int find_vpe_symbols(struct vpe * v, Elf_Shdr * sechdrs, +static int find_vpe_symbols(struct vpe *v, Elf_Shdr *sechdrs, unsigned int symindex, const char *strtab, struct module *mod) { @@ -804,16 +555,14 @@ static int find_vpe_symbols(struct vpe * v, Elf_Shdr * sechdrs, unsigned int i, n = sechdrs[symindex].sh_size / sizeof(Elf_Sym); for (i = 1; i < n; i++) { - if (strcmp(strtab + sym[i].st_name, "__start") == 0) { + if (strcmp(strtab + sym[i].st_name, "__start") == 0) v->__start = sym[i].st_value; - } - if (strcmp(strtab + sym[i].st_name, "vpe_shared") == 0) { + if (strcmp(strtab + sym[i].st_name, "vpe_shared") == 0) v->shared_ptr = (void *)sym[i].st_value; - } } - if ( (v->__start == 0) || (v->shared_ptr == NULL)) + if ((v->__start == 0) || (v->shared_ptr == NULL)) return -1; return 0; @@ -824,17 +573,17 @@ static int find_vpe_symbols(struct vpe * v, Elf_Shdr * sechdrs, * contents of the program (p)buffer performing relocatations/etc, free's it * when finished. */ -static int vpe_elfload(struct vpe * v) +static int vpe_elfload(struct vpe *v) { Elf_Ehdr *hdr; Elf_Shdr *sechdrs; long err = 0; char *secstrings, *strtab = NULL; unsigned int len, i, symindex = 0, strindex = 0, relocate = 0; - struct module mod; // so we can re-use the relocations code + struct module mod; /* so we can re-use the relocations code */ memset(&mod, 0, sizeof(struct module)); - strcpy(mod.name, "VPE loader"); + strscpy(mod.name, "VPE loader"); hdr = (Elf_Ehdr *) v->pbuffer; len = v->plen; @@ -845,8 +594,7 @@ static int vpe_elfload(struct vpe * v) || (hdr->e_type != ET_REL && hdr->e_type != ET_EXEC) || !elf_check_arch(hdr) || hdr->e_shentsize != sizeof(*sechdrs)) { - printk(KERN_WARNING - "VPE loader: program wrong arch or weird elf version\n"); + pr_warn("VPE loader: program wrong arch or weird elf version\n"); return -ENOEXEC; } @@ -855,8 +603,7 @@ static int vpe_elfload(struct vpe * v) relocate = 1; if (len < hdr->e_shoff + hdr->e_shnum * sizeof(Elf_Shdr)) { - printk(KERN_ERR "VPE loader: program length %u truncated\n", - len); + pr_err("VPE loader: program length %u truncated\n", len); return -ENOEXEC; } @@ -871,28 +618,30 @@ static int vpe_elfload(struct vpe * v) if (relocate) { for (i = 1; i < hdr->e_shnum; i++) { - if (sechdrs[i].sh_type != SHT_NOBITS - && len < sechdrs[i].sh_offset + sechdrs[i].sh_size) { - printk(KERN_ERR "VPE program length %u truncated\n", + if ((sechdrs[i].sh_type != SHT_NOBITS) && + (len < sechdrs[i].sh_offset + sechdrs[i].sh_size)) { + pr_err("VPE program length %u truncated\n", len); return -ENOEXEC; } /* Mark all sections sh_addr with their address in the temporary image. */ - sechdrs[i].sh_addr = (size_t) hdr + sechdrs[i].sh_offset; + sechdrs[i].sh_addr = (size_t) hdr + + sechdrs[i].sh_offset; /* Internal symbols and strings. */ if (sechdrs[i].sh_type == SHT_SYMTAB) { symindex = i; strindex = sechdrs[i].sh_link; - strtab = (char *)hdr + sechdrs[strindex].sh_offset; + strtab = (char *)hdr + + sechdrs[strindex].sh_offset; } } layout_sections(&mod, hdr, sechdrs, secstrings); } - v->load_addr = alloc_progmem(mod.core_size); + v->load_addr = alloc_progmem(mod.mem[MOD_TEXT].size); if (!v->load_addr) return -ENOMEM; @@ -913,8 +662,9 @@ static int vpe_elfload(struct vpe * v) /* Update sh_addr to point to copy in image. */ sechdrs[i].sh_addr = (unsigned long)dest; - printk(KERN_DEBUG " section sh_name %s sh_addr 0x%x\n", - secstrings + sechdrs[i].sh_name, sechdrs[i].sh_addr); + pr_debug(" section sh_name %s sh_addr 0x%x\n", + secstrings + sechdrs[i].sh_name, + sechdrs[i].sh_addr); } /* Fix up syms, so that st_value is a pointer to location. */ @@ -935,17 +685,18 @@ static int vpe_elfload(struct vpe * v) continue; if (sechdrs[i].sh_type == SHT_REL) - err = apply_relocations(sechdrs, strtab, symindex, i, - &mod); + err = apply_relocations(sechdrs, strtab, + symindex, i, &mod); else if (sechdrs[i].sh_type == SHT_RELA) - err = apply_relocate_add(sechdrs, strtab, symindex, i, - &mod); + err = apply_relocate_add(sechdrs, strtab, + symindex, i, &mod); if (err < 0) return err; } } else { - struct elf_phdr *phdr = (struct elf_phdr *) ((char *)hdr + hdr->e_phoff); + struct elf_phdr *phdr = (struct elf_phdr *) + ((char *)hdr + hdr->e_phoff); for (i = 0; i < hdr->e_phnum; i++) { if (phdr->p_type == PT_LOAD) { @@ -963,11 +714,15 @@ static int vpe_elfload(struct vpe * v) if (sechdrs[i].sh_type == SHT_SYMTAB) { symindex = i; strindex = sechdrs[i].sh_link; - strtab = (char *)hdr + sechdrs[strindex].sh_offset; + strtab = (char *)hdr + + sechdrs[strindex].sh_offset; - /* mark the symtab's address for when we try to find the - magic symbols */ - sechdrs[i].sh_addr = (size_t) hdr + sechdrs[i].sh_offset; + /* + * mark symtab's address for when we try + * to find the magic symbols + */ + sechdrs[i].sh_addr = (size_t) hdr + + sechdrs[i].sh_offset; } } } @@ -978,119 +733,60 @@ static int vpe_elfload(struct vpe * v) if ((find_vpe_symbols(v, sechdrs, symindex, strtab, &mod)) < 0) { if (v->__start == 0) { - printk(KERN_WARNING "VPE loader: program does not contain " - "a __start symbol\n"); + pr_warn("VPE loader: program does not contain a __start symbol\n"); return -ENOEXEC; } if (v->shared_ptr == NULL) - printk(KERN_WARNING "VPE loader: " - "program does not contain vpe_shared symbol.\n" - " Unable to use AMVP (AP/SP) facilities.\n"); + pr_warn("VPE loader: program does not contain vpe_shared symbol.\n" + " Unable to use AMVP (AP/SP) facilities.\n"); } - printk(" elf loaded\n"); + pr_info(" elf loaded\n"); return 0; } -static void cleanup_tc(struct tc *tc) -{ - unsigned long flags; - unsigned int mtflags, vpflags; - int tmp; - - local_irq_save(flags); - mtflags = dmt(); - vpflags = dvpe(); - /* Put MVPE's into 'configuration state' */ - set_c0_mvpcontrol(MVPCONTROL_VPC); - - settc(tc->index); - tmp = read_tc_c0_tcstatus(); - - /* mark not allocated and not dynamically allocatable */ - tmp &= ~(TCSTATUS_A | TCSTATUS_DA); - tmp |= TCSTATUS_IXMT; /* interrupt exempt */ - write_tc_c0_tcstatus(tmp); - - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - - /* bind it to anything other than VPE1 */ -// write_tc_c0_tcbind(read_tc_c0_tcbind() & ~TCBIND_CURVPE); // | TCBIND_CURVPE - - clear_c0_mvpcontrol(MVPCONTROL_VPC); - evpe(vpflags); - emt(mtflags); - local_irq_restore(flags); -} - -static int getcwd(char *buff, int size) -{ - mm_segment_t old_fs; - int ret; - - old_fs = get_fs(); - set_fs(KERNEL_DS); - - ret = sys_getcwd(buff, size); - - set_fs(old_fs); - - return ret; -} - /* checks VPE is unused and gets ready to load program */ static int vpe_open(struct inode *inode, struct file *filp) { enum vpe_state state; - struct vpe_notifications *not; + struct vpe_notifications *notifier; struct vpe *v; - int ret; - if (minor != iminor(inode)) { + if (VPE_MODULE_MINOR != iminor(inode)) { /* assume only 1 device at the moment. */ - pr_warning("VPE loader: only vpe1 is supported\n"); + pr_warn("VPE loader: only vpe1 is supported\n"); return -ENODEV; } - if ((v = get_vpe(tclimit)) == NULL) { - pr_warning("VPE loader: unable to get vpe\n"); + v = get_vpe(aprp_cpu_index()); + if (v == NULL) { + pr_warn("VPE loader: unable to get vpe\n"); return -ENODEV; } state = xchg(&v->state, VPE_STATE_INUSE); if (state != VPE_STATE_UNUSED) { - printk(KERN_DEBUG "VPE loader: tc in use dumping regs\n"); + pr_debug("VPE loader: tc in use dumping regs\n"); - list_for_each_entry(not, &v->notify, list) { - not->stop(tclimit); - } + list_for_each_entry(notifier, &v->notify, list) + notifier->stop(aprp_cpu_index()); release_progmem(v->load_addr); - cleanup_tc(get_tc(tclimit)); + cleanup_tc(get_tc(aprp_cpu_index())); } /* this of-course trashes what was there before... */ v->pbuffer = vmalloc(P_SIZE); if (!v->pbuffer) { - pr_warning("VPE loader: unable to allocate memory\n"); + pr_warn("VPE loader: unable to allocate memory\n"); return -ENOMEM; } v->plen = P_SIZE; v->load_addr = NULL; v->len = 0; - - v->uid = filp->f_cred->fsuid; - v->gid = filp->f_cred->fsgid; - - v->cwd[0] = 0; - ret = getcwd(v->cwd, VPE_PATH_MAX); - if (ret < 0) - printk(KERN_WARNING "VPE loader: open, getcwd returned %d\n", ret); - v->shared_ptr = NULL; v->__start = 0; @@ -1099,11 +795,12 @@ static int vpe_open(struct inode *inode, struct file *filp) static int vpe_release(struct inode *inode, struct file *filp) { +#ifdef CONFIG_MIPS_VPE_LOADER_MT struct vpe *v; Elf_Ehdr *hdr; int ret = 0; - v = get_vpe(tclimit); + v = get_vpe(aprp_cpu_index()); if (v == NULL) return -ENODEV; @@ -1112,11 +809,11 @@ static int vpe_release(struct inode *inode, struct file *filp) if (vpe_elfload(v) >= 0) { vpe_run(v); } else { - printk(KERN_WARNING "VPE loader: ELF load failed.\n"); + pr_warn("VPE loader: ELF load failed.\n"); ret = -ENOEXEC; } } else { - printk(KERN_WARNING "VPE loader: only elf files are supported\n"); + pr_warn("VPE loader: only elf files are supported\n"); ret = -ENOEXEC; } @@ -1132,24 +829,28 @@ static int vpe_release(struct inode *inode, struct file *filp) v->plen = 0; return ret; +#else + pr_warn("VPE loader: ELF load failed.\n"); + return -ENOEXEC; +#endif } -static ssize_t vpe_write(struct file *file, const char __user * buffer, - size_t count, loff_t * ppos) +static ssize_t vpe_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) { size_t ret = count; struct vpe *v; - if (iminor(file_inode(file)) != minor) + if (iminor(file_inode(file)) != VPE_MODULE_MINOR) return -ENODEV; - v = get_vpe(tclimit); + v = get_vpe(aprp_cpu_index()); + if (v == NULL) return -ENODEV; if ((count + v->len) > v->plen) { - printk(KERN_WARNING - "VPE loader: elf size too big. Perhaps strip uneeded symbols\n"); + pr_warn("VPE loader: elf size too big. Perhaps strip unneeded symbols\n"); return -ENOMEM; } @@ -1161,7 +862,7 @@ static ssize_t vpe_write(struct file *file, const char __user * buffer, return ret; } -static const struct file_operations vpe_fops = { +const struct file_operations vpe_fops = { .owner = THIS_MODULE, .open = vpe_open, .release = vpe_release, @@ -1169,417 +870,29 @@ static const struct file_operations vpe_fops = { .llseek = noop_llseek, }; -/* module wrapper entry points */ -/* give me a vpe */ -vpe_handle vpe_alloc(void) -{ - int i; - struct vpe *v; - - /* find a vpe */ - for (i = 1; i < MAX_VPES; i++) { - if ((v = get_vpe(i)) != NULL) { - v->state = VPE_STATE_INUSE; - return v; - } - } - return NULL; -} - -EXPORT_SYMBOL(vpe_alloc); - -/* start running from here */ -int vpe_start(vpe_handle vpe, unsigned long start) -{ - struct vpe *v = vpe; - - v->__start = start; - return vpe_run(v); -} - -EXPORT_SYMBOL(vpe_start); - -/* halt it for now */ -int vpe_stop(vpe_handle vpe) -{ - struct vpe *v = vpe; - struct tc *t; - unsigned int evpe_flags; - - evpe_flags = dvpe(); - - if ((t = list_entry(v->tc.next, struct tc, tc)) != NULL) { - - settc(t->index); - write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() & ~VPECONF0_VPA); - } - - evpe(evpe_flags); - - return 0; -} - -EXPORT_SYMBOL(vpe_stop); - -/* I've done with it thank you */ -int vpe_free(vpe_handle vpe) -{ - struct vpe *v = vpe; - struct tc *t; - unsigned int evpe_flags; - - if ((t = list_entry(v->tc.next, struct tc, tc)) == NULL) { - return -ENOEXEC; - } - - evpe_flags = dvpe(); - - /* Put MVPE's into 'configuration state' */ - set_c0_mvpcontrol(MVPCONTROL_VPC); - - settc(t->index); - write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() & ~VPECONF0_VPA); - - /* halt the TC */ - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - - /* mark the TC unallocated */ - write_tc_c0_tcstatus(read_tc_c0_tcstatus() & ~TCSTATUS_A); - - v->state = VPE_STATE_UNUSED; - - clear_c0_mvpcontrol(MVPCONTROL_VPC); - evpe(evpe_flags); - - return 0; -} - -EXPORT_SYMBOL(vpe_free); - void *vpe_get_shared(int index) { - struct vpe *v; + struct vpe *v = get_vpe(index); - if ((v = get_vpe(index)) == NULL) + if (v == NULL) return NULL; return v->shared_ptr; } - EXPORT_SYMBOL(vpe_get_shared); -int vpe_getuid(int index) -{ - struct vpe *v; - - if ((v = get_vpe(index)) == NULL) - return -1; - - return v->uid; -} - -EXPORT_SYMBOL(vpe_getuid); - -int vpe_getgid(int index) -{ - struct vpe *v; - - if ((v = get_vpe(index)) == NULL) - return -1; - - return v->gid; -} - -EXPORT_SYMBOL(vpe_getgid); - int vpe_notify(int index, struct vpe_notifications *notify) { - struct vpe *v; + struct vpe *v = get_vpe(index); - if ((v = get_vpe(index)) == NULL) + if (v == NULL) return -1; list_add(¬ify->list, &v->notify); return 0; } - EXPORT_SYMBOL(vpe_notify); -char *vpe_getcwd(int index) -{ - struct vpe *v; - - if ((v = get_vpe(index)) == NULL) - return NULL; - - return v->cwd; -} - -EXPORT_SYMBOL(vpe_getcwd); - -static ssize_t store_kill(struct device *dev, struct device_attribute *attr, - const char *buf, size_t len) -{ - struct vpe *vpe = get_vpe(tclimit); - struct vpe_notifications *not; - - list_for_each_entry(not, &vpe->notify, list) { - not->stop(tclimit); - } - - release_progmem(vpe->load_addr); - cleanup_tc(get_tc(tclimit)); - vpe_stop(vpe); - vpe_free(vpe); - - return len; -} - -static ssize_t show_ntcs(struct device *cd, struct device_attribute *attr, - char *buf) -{ - struct vpe *vpe = get_vpe(tclimit); - - return sprintf(buf, "%d\n", vpe->ntcs); -} - -static ssize_t store_ntcs(struct device *dev, struct device_attribute *attr, - const char *buf, size_t len) -{ - struct vpe *vpe = get_vpe(tclimit); - unsigned long new; - char *endp; - - new = simple_strtoul(buf, &endp, 0); - if (endp == buf) - goto out_einval; - - if (new == 0 || new > (hw_tcs - tclimit)) - goto out_einval; - - vpe->ntcs = new; - - return len; - -out_einval: - return -EINVAL; -} - -static struct device_attribute vpe_class_attributes[] = { - __ATTR(kill, S_IWUSR, NULL, store_kill), - __ATTR(ntcs, S_IRUGO | S_IWUSR, show_ntcs, store_ntcs), - {} -}; - -static void vpe_device_release(struct device *cd) -{ - kfree(cd); -} - -struct class vpe_class = { - .name = "vpe", - .owner = THIS_MODULE, - .dev_release = vpe_device_release, - .dev_attrs = vpe_class_attributes, -}; - -struct device vpe_device; - -static int __init vpe_module_init(void) -{ - unsigned int mtflags, vpflags; - unsigned long flags, val; - struct vpe *v = NULL; - struct tc *t; - int tc, err; - - if (!cpu_has_mipsmt) { - printk("VPE loader: not a MIPS MT capable processor\n"); - return -ENODEV; - } - - if (vpelimit == 0) { - printk(KERN_WARNING "No VPEs reserved for AP/SP, not " - "initializing VPE loader.\nPass maxvpes=<n> argument as " - "kernel argument\n"); - - return -ENODEV; - } - - if (tclimit == 0) { - printk(KERN_WARNING "No TCs reserved for AP/SP, not " - "initializing VPE loader.\nPass maxtcs=<n> argument as " - "kernel argument\n"); - - return -ENODEV; - } - - major = register_chrdev(0, module_name, &vpe_fops); - if (major < 0) { - printk("VPE loader: unable to register character device\n"); - return major; - } - - err = class_register(&vpe_class); - if (err) { - printk(KERN_ERR "vpe_class registration failed\n"); - goto out_chrdev; - } - - device_initialize(&vpe_device); - vpe_device.class = &vpe_class, - vpe_device.parent = NULL, - dev_set_name(&vpe_device, "vpe1"); - vpe_device.devt = MKDEV(major, minor); - err = device_add(&vpe_device); - if (err) { - printk(KERN_ERR "Adding vpe_device failed\n"); - goto out_class; - } - - local_irq_save(flags); - mtflags = dmt(); - vpflags = dvpe(); - - /* Put MVPE's into 'configuration state' */ - set_c0_mvpcontrol(MVPCONTROL_VPC); - - /* dump_mtregs(); */ - - val = read_c0_mvpconf0(); - hw_tcs = (val & MVPCONF0_PTC) + 1; - hw_vpes = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; - - for (tc = tclimit; tc < hw_tcs; tc++) { - /* - * Must re-enable multithreading temporarily or in case we - * reschedule send IPIs or similar we might hang. - */ - clear_c0_mvpcontrol(MVPCONTROL_VPC); - evpe(vpflags); - emt(mtflags); - local_irq_restore(flags); - t = alloc_tc(tc); - if (!t) { - err = -ENOMEM; - goto out; - } - - local_irq_save(flags); - mtflags = dmt(); - vpflags = dvpe(); - set_c0_mvpcontrol(MVPCONTROL_VPC); - - /* VPE's */ - if (tc < hw_tcs) { - settc(tc); - - if ((v = alloc_vpe(tc)) == NULL) { - printk(KERN_WARNING "VPE: unable to allocate VPE\n"); - - goto out_reenable; - } - - v->ntcs = hw_tcs - tclimit; - - /* add the tc to the list of this vpe's tc's. */ - list_add(&t->tc, &v->tc); - - /* deactivate all but vpe0 */ - if (tc >= tclimit) { - unsigned long tmp = read_vpe_c0_vpeconf0(); - - tmp &= ~VPECONF0_VPA; - - /* master VPE */ - tmp |= VPECONF0_MVP; - write_vpe_c0_vpeconf0(tmp); - } - - /* disable multi-threading with TC's */ - write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() & ~VPECONTROL_TE); - - if (tc >= vpelimit) { - /* - * Set config to be the same as vpe0, - * particularly kseg0 coherency alg - */ - write_vpe_c0_config(read_c0_config()); - } - } - - /* TC's */ - t->pvpe = v; /* set the parent vpe */ - - if (tc >= tclimit) { - unsigned long tmp; - - settc(tc); - - /* Any TC that is bound to VPE0 gets left as is - in case - we are running SMTC on VPE0. A TC that is bound to any - other VPE gets bound to VPE0, ideally I'd like to make - it homeless but it doesn't appear to let me bind a TC - to a non-existent VPE. Which is perfectly reasonable. - - The (un)bound state is visible to an EJTAG probe so may - notify GDB... - */ - - if (((tmp = read_tc_c0_tcbind()) & TCBIND_CURVPE)) { - /* tc is bound >vpe0 */ - write_tc_c0_tcbind(tmp & ~TCBIND_CURVPE); - - t->pvpe = get_vpe(0); /* set the parent vpe */ - } - - /* halt the TC */ - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - - tmp = read_tc_c0_tcstatus(); - - /* mark not activated and not dynamically allocatable */ - tmp &= ~(TCSTATUS_A | TCSTATUS_DA); - tmp |= TCSTATUS_IXMT; /* interrupt exempt */ - write_tc_c0_tcstatus(tmp); - } - } - -out_reenable: - /* release config state */ - clear_c0_mvpcontrol(MVPCONTROL_VPC); - - evpe(vpflags); - emt(mtflags); - local_irq_restore(flags); - - return 0; - -out_class: - class_unregister(&vpe_class); -out_chrdev: - unregister_chrdev(major, module_name); - -out: - return err; -} - -static void __exit vpe_module_exit(void) -{ - struct vpe *v, *n; - - device_del(&vpe_device); - unregister_chrdev(major, module_name); - - /* No locking needed here */ - list_for_each_entry_safe(v, n, &vpecontrol.vpe_list, list) { - if (v->state != VPE_STATE_UNUSED) - release_vpe(v); - } -} - module_init(vpe_module_init); module_exit(vpe_module_exit); MODULE_DESCRIPTION("MIPS VPE Loader"); diff --git a/arch/mips/kernel/watch.c b/arch/mips/kernel/watch.c index cbdc4de85bb4..c9263b95cb2e 100644 --- a/arch/mips/kernel/watch.c +++ b/arch/mips/kernel/watch.c @@ -15,27 +15,30 @@ * Install the watch registers for the current thread. A maximum of * four registers are installed although the machine may have more. */ -void mips_install_watch_registers(void) +void mips_install_watch_registers(struct task_struct *t) { - struct mips3264_watch_reg_state *watches = - ¤t->thread.watch.mips3264; + struct mips3264_watch_reg_state *watches = &t->thread.watch.mips3264; + unsigned int watchhi = MIPS_WATCHHI_G | /* Trap all ASIDs */ + MIPS_WATCHHI_IRW; /* Clear result bits */ + switch (current_cpu_data.watch_reg_use_cnt) { default: BUG(); case 4: write_c0_watchlo3(watches->watchlo[3]); - /* Write 1 to the I, R, and W bits to clear them, and - 1 to G so all ASIDs are trapped. */ - write_c0_watchhi3(0x40000007 | watches->watchhi[3]); + write_c0_watchhi3(watchhi | watches->watchhi[3]); + fallthrough; case 3: write_c0_watchlo2(watches->watchlo[2]); - write_c0_watchhi2(0x40000007 | watches->watchhi[2]); + write_c0_watchhi2(watchhi | watches->watchhi[2]); + fallthrough; case 2: write_c0_watchlo1(watches->watchlo[1]); - write_c0_watchhi1(0x40000007 | watches->watchhi[1]); + write_c0_watchhi1(watchhi | watches->watchhi[1]); + fallthrough; case 1: write_c0_watchlo0(watches->watchlo[0]); - write_c0_watchhi0(0x40000007 | watches->watchhi[0]); + write_c0_watchhi0(watchhi | watches->watchhi[0]); } } @@ -48,26 +51,31 @@ void mips_read_watch_registers(void) { struct mips3264_watch_reg_state *watches = ¤t->thread.watch.mips3264; + unsigned int watchhi_mask = MIPS_WATCHHI_MASK | MIPS_WATCHHI_IRW; + switch (current_cpu_data.watch_reg_use_cnt) { default: BUG(); case 4: - watches->watchhi[3] = (read_c0_watchhi3() & 0x0fff); + watches->watchhi[3] = (read_c0_watchhi3() & watchhi_mask); + fallthrough; case 3: - watches->watchhi[2] = (read_c0_watchhi2() & 0x0fff); + watches->watchhi[2] = (read_c0_watchhi2() & watchhi_mask); + fallthrough; case 2: - watches->watchhi[1] = (read_c0_watchhi1() & 0x0fff); + watches->watchhi[1] = (read_c0_watchhi1() & watchhi_mask); + fallthrough; case 1: - watches->watchhi[0] = (read_c0_watchhi0() & 0x0fff); + watches->watchhi[0] = (read_c0_watchhi0() & watchhi_mask); } if (current_cpu_data.watch_reg_use_cnt == 1 && - (watches->watchhi[0] & 7) == 0) { + (watches->watchhi[0] & MIPS_WATCHHI_IRW) == 0) { /* Pathological case of release 1 architecture that * doesn't set the condition bits. We assume that * since we got here, the watch condition was met and * signal that the conditions requested in watchlo * were met. */ - watches->watchhi[0] |= (watches->watchlo[0] & 7); + watches->watchhi[0] |= (watches->watchlo[0] & MIPS_WATCHHI_IRW); } } @@ -83,24 +91,31 @@ void mips_clear_watch_registers(void) BUG(); case 8: write_c0_watchlo7(0); + fallthrough; case 7: write_c0_watchlo6(0); + fallthrough; case 6: write_c0_watchlo5(0); + fallthrough; case 5: write_c0_watchlo4(0); + fallthrough; case 4: write_c0_watchlo3(0); + fallthrough; case 3: write_c0_watchlo2(0); + fallthrough; case 2: write_c0_watchlo1(0); + fallthrough; case 1: write_c0_watchlo0(0); } } -__cpuinit void mips_probe_watch_registers(struct cpuinfo_mips *c) +void mips_probe_watch_registers(struct cpuinfo_mips *c) { unsigned int t; @@ -110,86 +125,86 @@ __cpuinit void mips_probe_watch_registers(struct cpuinfo_mips *c) * Check which of the I,R and W bits are supported, then * disable the register. */ - write_c0_watchlo0(7); + write_c0_watchlo0(MIPS_WATCHLO_IRW); back_to_back_c0_hazard(); t = read_c0_watchlo0(); write_c0_watchlo0(0); - c->watch_reg_masks[0] = t & 7; + c->watch_reg_masks[0] = t & MIPS_WATCHLO_IRW; /* Write the mask bits and read them back to determine which * can be used. */ c->watch_reg_count = 1; c->watch_reg_use_cnt = 1; t = read_c0_watchhi0(); - write_c0_watchhi0(t | 0xff8); + write_c0_watchhi0(t | MIPS_WATCHHI_MASK); back_to_back_c0_hazard(); t = read_c0_watchhi0(); - c->watch_reg_masks[0] |= (t & 0xff8); - if ((t & 0x80000000) == 0) + c->watch_reg_masks[0] |= (t & MIPS_WATCHHI_MASK); + if ((t & MIPS_WATCHHI_M) == 0) return; - write_c0_watchlo1(7); + write_c0_watchlo1(MIPS_WATCHLO_IRW); back_to_back_c0_hazard(); t = read_c0_watchlo1(); write_c0_watchlo1(0); - c->watch_reg_masks[1] = t & 7; + c->watch_reg_masks[1] = t & MIPS_WATCHLO_IRW; c->watch_reg_count = 2; c->watch_reg_use_cnt = 2; t = read_c0_watchhi1(); - write_c0_watchhi1(t | 0xff8); + write_c0_watchhi1(t | MIPS_WATCHHI_MASK); back_to_back_c0_hazard(); t = read_c0_watchhi1(); - c->watch_reg_masks[1] |= (t & 0xff8); - if ((t & 0x80000000) == 0) + c->watch_reg_masks[1] |= (t & MIPS_WATCHHI_MASK); + if ((t & MIPS_WATCHHI_M) == 0) return; - write_c0_watchlo2(7); + write_c0_watchlo2(MIPS_WATCHLO_IRW); back_to_back_c0_hazard(); t = read_c0_watchlo2(); write_c0_watchlo2(0); - c->watch_reg_masks[2] = t & 7; + c->watch_reg_masks[2] = t & MIPS_WATCHLO_IRW; c->watch_reg_count = 3; c->watch_reg_use_cnt = 3; t = read_c0_watchhi2(); - write_c0_watchhi2(t | 0xff8); + write_c0_watchhi2(t | MIPS_WATCHHI_MASK); back_to_back_c0_hazard(); t = read_c0_watchhi2(); - c->watch_reg_masks[2] |= (t & 0xff8); - if ((t & 0x80000000) == 0) + c->watch_reg_masks[2] |= (t & MIPS_WATCHHI_MASK); + if ((t & MIPS_WATCHHI_M) == 0) return; - write_c0_watchlo3(7); + write_c0_watchlo3(MIPS_WATCHLO_IRW); back_to_back_c0_hazard(); t = read_c0_watchlo3(); write_c0_watchlo3(0); - c->watch_reg_masks[3] = t & 7; + c->watch_reg_masks[3] = t & MIPS_WATCHLO_IRW; c->watch_reg_count = 4; c->watch_reg_use_cnt = 4; t = read_c0_watchhi3(); - write_c0_watchhi3(t | 0xff8); + write_c0_watchhi3(t | MIPS_WATCHHI_MASK); back_to_back_c0_hazard(); t = read_c0_watchhi3(); - c->watch_reg_masks[3] |= (t & 0xff8); - if ((t & 0x80000000) == 0) + c->watch_reg_masks[3] |= (t & MIPS_WATCHHI_MASK); + if ((t & MIPS_WATCHHI_M) == 0) return; /* We use at most 4, but probe and report up to 8. */ c->watch_reg_count = 5; t = read_c0_watchhi4(); - if ((t & 0x80000000) == 0) + if ((t & MIPS_WATCHHI_M) == 0) return; c->watch_reg_count = 6; t = read_c0_watchhi5(); - if ((t & 0x80000000) == 0) + if ((t & MIPS_WATCHHI_M) == 0) return; c->watch_reg_count = 7; t = read_c0_watchhi6(); - if ((t & 0x80000000) == 0) + if ((t & MIPS_WATCHHI_M) == 0) return; c->watch_reg_count = 8; |