diff options
Diffstat (limited to 'arch/arm/kernel')
56 files changed, 543 insertions, 4197 deletions
diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index d53f56d6f840..afc9de7ef9a1 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -40,6 +40,7 @@ obj-y += entry-armv.o endif obj-$(CONFIG_MMU) += bugs.o +obj-$(CONFIG_OF) += cacheinfo.o obj-$(CONFIG_CPU_IDLE) += cpuidle.o obj-$(CONFIG_ISA_DMA_API) += dma.o obj-$(CONFIG_FIQ) += fiq.o fiqasm.o @@ -59,7 +60,8 @@ obj-$(CONFIG_FUNCTION_TRACER) += entry-ftrace.o obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o insn.o patch.o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o insn.o patch.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o insn.o patch.o -obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o +obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o +obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o # Main staffs in KPROBES are in arch/arm/probes/ . obj-$(CONFIG_KPROBES) += patch.o insn.o obj-$(CONFIG_OABI_COMPAT) += sys_oabi-compat.o @@ -75,12 +77,8 @@ obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o obj-$(CONFIG_CPU_XSCALE) += xscale-cp0.o obj-$(CONFIG_CPU_XSC3) += xscale-cp0.o obj-$(CONFIG_CPU_MOHAWK) += xscale-cp0.o -obj-$(CONFIG_CPU_PJ4) += pj4-cp0.o -obj-$(CONFIG_CPU_PJ4B) += pj4-cp0.o obj-$(CONFIG_IWMMXT) += iwmmxt.o obj-$(CONFIG_PERF_EVENTS) += perf_regs.o perf_callchain.o -obj-$(CONFIG_HW_PERF_EVENTS) += perf_event_xscale.o perf_event_v6.o \ - perf_event_v7.o AFLAGS_iwmmxt.o := -Wa,-mcpu=iwmmxt obj-$(CONFIG_ARM_CPU_TOPOLOGY) += topology.o obj-$(CONFIG_VDSO) += vdso.o @@ -106,4 +104,4 @@ obj-$(CONFIG_HAVE_ARM_SMCCC) += smccc-call.o obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += spectre.o -extra-y := vmlinux.lds +always-$(KBUILD_BUILTIN) := vmlinux.lds diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 2c8d76fd7c66..2101938d27fc 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -7,6 +7,8 @@ * This code generates raw asm output which is post-processed to extract * and format the required data. */ +#define COMPILE_OFFSETS + #include <linux/compiler.h> #include <linux/sched.h> #include <linux/mm.h> @@ -17,14 +19,16 @@ #include <asm/glue-pf.h> #include <asm/mach/arch.h> #include <asm/thread_info.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/mpu.h> #include <asm/procinfo.h> #include <asm/suspend.h> -#include <asm/vdso_datapage.h> #include <asm/hardware/cache-l2x0.h> #include <linux/kbuild.h> #include <linux/arm-smccc.h> + +#include <vdso/datapage.h> + #include "signal.h" /* @@ -47,7 +51,6 @@ int main(void) DEFINE(TI_CPU_DOMAIN, offsetof(struct thread_info, cpu_domain)); DEFINE(TI_CPU_SAVE, offsetof(struct thread_info, cpu_context)); DEFINE(TI_ABI_SYSCALL, offsetof(struct thread_info, abi_syscall)); - DEFINE(TI_USED_CP, offsetof(struct thread_info, used_cp)); DEFINE(TI_TP_VALUE, offsetof(struct thread_info, tp_value)); DEFINE(TI_FPSTATE, offsetof(struct thread_info, fpstate)); #ifdef CONFIG_VFP @@ -56,6 +59,7 @@ int main(void) DEFINE(VFP_CPU, offsetof(union vfp_state, hard.cpu)); #endif #endif + DEFINE(SOFTIRQ_DISABLE_OFFSET,SOFTIRQ_DISABLE_OFFSET); #ifdef CONFIG_ARM_THUMBEE DEFINE(TI_THUMBEE_STATE, offsetof(struct thread_info, thumbee_state)); #endif @@ -83,6 +87,7 @@ int main(void) DEFINE(S_OLD_R0, offsetof(struct pt_regs, ARM_ORIG_r0)); DEFINE(PT_REGS_SIZE, sizeof(struct pt_regs)); DEFINE(SVC_DACR, offsetof(struct svc_pt_regs, dacr)); + DEFINE(SVC_TTBCR, offsetof(struct svc_pt_regs, ttbcr)); DEFINE(SVC_REGS_SIZE, sizeof(struct svc_pt_regs)); BLANK(); DEFINE(SIGFRAME_RC3_OFFSET, offsetof(struct sigframe, retcode[3])); @@ -150,10 +155,6 @@ int main(void) DEFINE(CACHE_WRITEBACK_ORDER, __CACHE_WRITEBACK_ORDER); DEFINE(CACHE_WRITEBACK_GRANULE, __CACHE_WRITEBACK_GRANULE); BLANK(); -#ifdef CONFIG_VDSO - DEFINE(VDSO_DATA_SIZE, sizeof(union vdso_data_store)); -#endif - BLANK(); #ifdef CONFIG_ARM_MPU DEFINE(MPU_RNG_INFO_RNGS, offsetof(struct mpu_rgn_info, rgns)); DEFINE(MPU_RNG_INFO_USED, offsetof(struct mpu_rgn_info, used)); diff --git a/arch/arm/kernel/atags_parse.c b/arch/arm/kernel/atags_parse.c index 373b61f9a4f0..4ec591bde3df 100644 --- a/arch/arm/kernel/atags_parse.c +++ b/arch/arm/kernel/atags_parse.c @@ -69,18 +69,18 @@ static int __init parse_tag_mem32(const struct tag *tag) __tagtable(ATAG_MEM, parse_tag_mem32); -#if defined(CONFIG_VGA_CONSOLE) || defined(CONFIG_DUMMY_CONSOLE) +#if defined(CONFIG_ARCH_FOOTBRIDGE) && defined(CONFIG_VGA_CONSOLE) static int __init parse_tag_videotext(const struct tag *tag) { - screen_info.orig_x = tag->u.videotext.x; - screen_info.orig_y = tag->u.videotext.y; - screen_info.orig_video_page = tag->u.videotext.video_page; - screen_info.orig_video_mode = tag->u.videotext.video_mode; - screen_info.orig_video_cols = tag->u.videotext.video_cols; - screen_info.orig_video_ega_bx = tag->u.videotext.video_ega_bx; - screen_info.orig_video_lines = tag->u.videotext.video_lines; - screen_info.orig_video_isVGA = tag->u.videotext.video_isvga; - screen_info.orig_video_points = tag->u.videotext.video_points; + vgacon_screen_info.orig_x = tag->u.videotext.x; + vgacon_screen_info.orig_y = tag->u.videotext.y; + vgacon_screen_info.orig_video_page = tag->u.videotext.video_page; + vgacon_screen_info.orig_video_mode = tag->u.videotext.video_mode; + vgacon_screen_info.orig_video_cols = tag->u.videotext.video_cols; + vgacon_screen_info.orig_video_ega_bx = tag->u.videotext.video_ega_bx; + vgacon_screen_info.orig_video_lines = tag->u.videotext.video_lines; + vgacon_screen_info.orig_video_isVGA = tag->u.videotext.video_isvga; + vgacon_screen_info.orig_video_points = tag->u.videotext.video_points; return 0; } @@ -127,7 +127,7 @@ static int __init parse_tag_cmdline(const struct tag *tag) #elif defined(CONFIG_CMDLINE_FORCE) pr_warn("Ignoring tag cmdline (using the default kernel command line)\n"); #else - strlcpy(default_command_line, tag->u.cmdline.cmdline, + strscpy(default_command_line, tag->u.cmdline.cmdline, COMMAND_LINE_SIZE); #endif return 0; @@ -224,7 +224,7 @@ setup_machine_tags(void *atags_vaddr, unsigned int machine_nr) } /* parse_early_param needs a boot_command_line */ - strlcpy(boot_command_line, from, COMMAND_LINE_SIZE); + strscpy(boot_command_line, from, COMMAND_LINE_SIZE); return mdesc; } diff --git a/arch/arm/kernel/atags_proc.c b/arch/arm/kernel/atags_proc.c index 3ec2afe78423..cd09f8ab93e3 100644 --- a/arch/arm/kernel/atags_proc.c +++ b/arch/arm/kernel/atags_proc.c @@ -7,7 +7,7 @@ struct buffer { size_t size; - char data[]; + char data[] __counted_by(size); }; static ssize_t atags_read(struct file *file, char __user *buf, @@ -54,7 +54,7 @@ static int __init init_atags_procfs(void) WARN_ON(tag->hdr.tag != ATAG_NONE); - b = kmalloc(sizeof(*b) + size, GFP_KERNEL); + b = kmalloc(struct_size(b, data, size), GFP_KERNEL); if (!b) goto nomem; diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c index e7ef2b5bea9c..b5793e8fbdc1 100644 --- a/arch/arm/kernel/bios32.c +++ b/arch/arm/kernel/bios32.c @@ -10,6 +10,7 @@ #include <linux/kernel.h> #include <linux/pci.h> #include <linux/slab.h> +#include <linux/string_choices.h> #include <linux/init.h> #include <linux/io.h> @@ -142,15 +143,15 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_WINBOND2, PCI_DEVICE_ID_WINBOND2_89C940F, */ static void pci_fixup_dec21285(struct pci_dev *dev) { - int i; - if (dev->devfn == 0) { + struct resource *r; + dev->class &= 0xff; dev->class |= PCI_CLASS_BRIDGE_HOST << 8; - for (i = 0; i < PCI_NUM_RESOURCES; i++) { - dev->resource[i].start = 0; - dev->resource[i].end = 0; - dev->resource[i].flags = 0; + pci_dev_for_each_resource(dev, r) { + r->start = 0; + r->end = 0; + r->flags = 0; } } } @@ -162,13 +163,11 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_DEC, PCI_DEVICE_ID_DEC_21285, pci_fixup_d static void pci_fixup_ide_bases(struct pci_dev *dev) { struct resource *r; - int i; if ((dev->class >> 8) != PCI_CLASS_STORAGE_IDE) return; - for (i = 0; i < PCI_NUM_RESOURCES; i++) { - r = dev->resource + i; + pci_dev_for_each_resource(dev, r) { if ((r->start & ~0x80) == 0x374) { r->start |= 2; r->end = r->start; @@ -339,8 +338,8 @@ void pcibios_fixup_bus(struct pci_bus *bus) /* * Report what we did for this bus */ - pr_info("PCI: bus%d: Fast back to back transfers %sabled\n", - bus->number, (features & PCI_COMMAND_FAST_BACK) ? "en" : "dis"); + pr_info("PCI: bus%d: Fast back to back transfers %s\n", + bus->number, str_enabled_disabled(features & PCI_COMMAND_FAST_BACK)); } EXPORT_SYMBOL(pcibios_fixup_bus); diff --git a/arch/arm/kernel/bugs.c b/arch/arm/kernel/bugs.c index 14c8dbbb7d2d..087bce6ec8e9 100644 --- a/arch/arm/kernel/bugs.c +++ b/arch/arm/kernel/bugs.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/init.h> +#include <linux/cpu.h> #include <asm/bugs.h> #include <asm/proc-fns.h> @@ -11,7 +12,7 @@ void check_other_bugs(void) #endif } -void __init check_bugs(void) +void __init arch_cpu_finalize_init(void) { check_writebuffer_bugs(); check_other_bugs(); diff --git a/arch/arm/kernel/cacheinfo.c b/arch/arm/kernel/cacheinfo.c new file mode 100644 index 000000000000..e1469b641780 --- /dev/null +++ b/arch/arm/kernel/cacheinfo.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * ARM cacheinfo support + * + * Copyright (C) 2023 Linaro Ltd. + * Copyright (C) 2015 ARM Ltd. + * All Rights Reserved + */ + +#include <linux/bitfield.h> +#include <linux/cacheinfo.h> +#include <linux/of.h> + +#include <asm/cachetype.h> +#include <asm/cputype.h> +#include <asm/system_info.h> + +/* Ctypen, bits[3(n - 1) + 2 : 3(n - 1)], for n = 1 to 7 */ +#define CLIDR_CTYPE_SHIFT(level) (3 * (level - 1)) +#define CLIDR_CTYPE_MASK(level) (7 << CLIDR_CTYPE_SHIFT(level)) +#define CLIDR_CTYPE(clidr, level) \ + (((clidr) & CLIDR_CTYPE_MASK(level)) >> CLIDR_CTYPE_SHIFT(level)) + +#define MAX_CACHE_LEVEL 7 /* Max 7 level supported */ + +#define CTR_FORMAT_MASK GENMASK(31, 29) +#define CTR_FORMAT_ARMV6 0 +#define CTR_FORMAT_ARMV7 4 +#define CTR_CWG_MASK GENMASK(27, 24) +#define CTR_DSIZE_LEN_MASK GENMASK(13, 12) +#define CTR_ISIZE_LEN_MASK GENMASK(1, 0) + +/* Also valid for v7m */ +static inline int cache_line_size_cp15(void) +{ + u32 ctr = read_cpuid_cachetype(); + u32 format = FIELD_GET(CTR_FORMAT_MASK, ctr); + + if (format == CTR_FORMAT_ARMV7) { + u32 cwg = FIELD_GET(CTR_CWG_MASK, ctr); + + return cwg ? 4 << cwg : ARCH_DMA_MINALIGN; + } else if (WARN_ON_ONCE(format != CTR_FORMAT_ARMV6)) { + return ARCH_DMA_MINALIGN; + } + + return 8 << max(FIELD_GET(CTR_ISIZE_LEN_MASK, ctr), + FIELD_GET(CTR_DSIZE_LEN_MASK, ctr)); +} + +int cache_line_size(void) +{ + if (coherency_max_size != 0) + return coherency_max_size; + + /* CP15 is optional / implementation defined before ARMv6 */ + if (cpu_architecture() < CPU_ARCH_ARMv6) + return ARCH_DMA_MINALIGN; + + return cache_line_size_cp15(); +} +EXPORT_SYMBOL_GPL(cache_line_size); + +static inline enum cache_type get_cache_type(int level) +{ + u32 clidr; + + if (level > MAX_CACHE_LEVEL) + return CACHE_TYPE_NOCACHE; + + clidr = read_clidr(); + + return CLIDR_CTYPE(clidr, level); +} + +static void ci_leaf_init(struct cacheinfo *this_leaf, + enum cache_type type, unsigned int level) +{ + this_leaf->level = level; + this_leaf->type = type; +} + +static int detect_cache_level(unsigned int *level_p, unsigned int *leaves_p) +{ + unsigned int ctype, level, leaves; + u32 ctr, format; + + /* CLIDR is not present before ARMv7/v7m */ + if (cpu_architecture() < CPU_ARCH_ARMv7) + return -EOPNOTSUPP; + + /* Don't try reading CLIDR if CTR declares old format */ + ctr = read_cpuid_cachetype(); + format = FIELD_GET(CTR_FORMAT_MASK, ctr); + if (format != CTR_FORMAT_ARMV7) + return -EOPNOTSUPP; + + for (level = 1, leaves = 0; level <= MAX_CACHE_LEVEL; level++) { + ctype = get_cache_type(level); + if (ctype == CACHE_TYPE_NOCACHE) { + level--; + break; + } + /* Separate instruction and data caches */ + leaves += (ctype == CACHE_TYPE_SEPARATE) ? 2 : 1; + } + + *level_p = level; + *leaves_p = leaves; + + return 0; +} + +int early_cache_level(unsigned int cpu) +{ + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + + return detect_cache_level(&this_cpu_ci->num_levels, &this_cpu_ci->num_leaves); +} + +int init_cache_level(unsigned int cpu) +{ + unsigned int level, leaves; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + int fw_level; + int ret; + + ret = detect_cache_level(&level, &leaves); + if (ret) + return ret; + + fw_level = of_find_last_cache_level(cpu); + + if (level < fw_level) { + /* + * some external caches not specified in CLIDR_EL1 + * the information may be available in the device tree + * only unified external caches are considered here + */ + leaves += (fw_level - level); + level = fw_level; + } + + this_cpu_ci->num_levels = level; + this_cpu_ci->num_leaves = leaves; + return 0; +} + +int populate_cache_leaves(unsigned int cpu) +{ + unsigned int level, idx; + enum cache_type type; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct cacheinfo *this_leaf = this_cpu_ci->info_list; + unsigned int arch = cpu_architecture(); + + /* CLIDR is not present before ARMv7/v7m */ + if (arch < CPU_ARCH_ARMv7) + return -EOPNOTSUPP; + + for (idx = 0, level = 1; level <= this_cpu_ci->num_levels && + idx < this_cpu_ci->num_leaves; idx++, level++) { + type = get_cache_type(level); + if (type == CACHE_TYPE_SEPARATE) { + ci_leaf_init(this_leaf++, CACHE_TYPE_DATA, level); + ci_leaf_init(this_leaf++, CACHE_TYPE_INST, level); + } else { + ci_leaf_init(this_leaf++, type, level); + } + } + + return 0; +} diff --git a/arch/arm/kernel/cpuidle.c b/arch/arm/kernel/cpuidle.c index e1684623e1b2..fba1f8bb03b5 100644 --- a/arch/arm/kernel/cpuidle.c +++ b/arch/arm/kernel/cpuidle.c @@ -5,7 +5,6 @@ #include <linux/cpuidle.h> #include <linux/of.h> -#include <linux/of_device.h> #include <asm/cpuidle.h> extern struct of_cpuidle_method __cpuidle_method_of_table[]; @@ -26,8 +25,8 @@ static struct cpuidle_ops cpuidle_ops[NR_CPUS] __ro_after_init; * * Returns the index passed as parameter */ -int arm_cpuidle_simple_enter(struct cpuidle_device *dev, - struct cpuidle_driver *drv, int index) +__cpuidle int arm_cpuidle_simple_enter(struct cpuidle_device *dev, struct + cpuidle_driver *drv, int index) { cpu_do_idle(); diff --git a/arch/arm/kernel/devtree.c b/arch/arm/kernel/devtree.c index 264827281113..3b78966e750a 100644 --- a/arch/arm/kernel/devtree.c +++ b/arch/arm/kernel/devtree.c @@ -13,7 +13,6 @@ #include <linux/of.h> #include <linux/of_fdt.h> #include <linux/of_irq.h> -#include <linux/of_platform.h> #include <linux/smp.h> #include <asm/cputype.h> @@ -201,7 +200,7 @@ const struct machine_desc * __init setup_machine_fdt(void *dt_virt) mdesc_best = &__mach_desc_GENERIC_DT; - if (!dt_virt || !early_init_dt_verify(dt_virt)) + if (!dt_virt || !early_init_dt_verify(dt_virt, __pa(dt_virt))) return NULL; mdesc = of_flat_dt_match_machine(mdesc_best, arch_get_next_mach); diff --git a/arch/arm/kernel/efi.c b/arch/arm/kernel/efi.c index 882104f43b3b..6f9ec7d28a71 100644 --- a/arch/arm/kernel/efi.c +++ b/arch/arm/kernel/efi.c @@ -5,6 +5,8 @@ #include <linux/efi.h> #include <linux/memblock.h> +#include <linux/screen_info.h> + #include <asm/efi.h> #include <asm/mach/map.h> #include <asm/mmu_context.h> @@ -23,7 +25,8 @@ static int __init set_permissions(pte_t *ptep, unsigned long addr, void *data) } int __init efi_set_mapping_permissions(struct mm_struct *mm, - efi_memory_desc_t *md) + efi_memory_desc_t *md, + bool ignored) { unsigned long base, size; @@ -71,7 +74,7 @@ int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) * If stricter permissions were specified, apply them now. */ if (md->attribute & (EFI_MEMORY_RO | EFI_MEMORY_XP)) - return efi_set_mapping_permissions(mm, md); + return efi_set_mapping_permissions(mm, md, false); return 0; } @@ -120,12 +123,6 @@ void __init arm_efi_init(void) { efi_init(); - if (screen_info.orig_video_isVGA == VIDEO_TYPE_EFI) { - /* dummycon on ARM needs non-zero values for columns/lines */ - screen_info.orig_video_cols = 80; - screen_info.orig_video_lines = 25; - } - /* ARM does not permit early mappings to persist across paging_init() */ efi_memmap_unmap(); diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index c39303e5c234..ef6a657c8d13 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -15,7 +15,7 @@ #include <linux/init.h> #include <asm/assembler.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/glue-df.h> #include <asm/glue-pf.h> #include <asm/vfpmacros.h> @@ -25,10 +25,17 @@ #include <asm/tls.h> #include <asm/system_info.h> #include <asm/uaccess-asm.h> +#include <asm/kasan_def.h> #include "entry-header.S" #include <asm/probes.h> +#ifdef CONFIG_HAVE_LD_DEAD_CODE_DATA_ELIMINATION +#define RELOC_TEXT_NONE .reloc .text, R_ARM_NONE, . +#else +#define RELOC_TEXT_NONE +#endif + /* * Interrupt handling. */ @@ -446,258 +453,26 @@ ENDPROC(__irq_usr) __und_usr: usr_entry uaccess=0 - mov r2, r4 - mov r3, r5 - - @ r2 = regs->ARM_pc, which is either 2 or 4 bytes ahead of the - @ faulting instruction depending on Thumb mode. - @ r3 = regs->ARM_cpsr - @ - @ The emulation code returns using r9 if it has emulated the - @ instruction, or the more conventional lr if we are to treat - @ this as a real undefined instruction - @ - badr r9, ret_from_exception - @ IRQs must be enabled before attempting to read the instruction from @ user space since that could cause a page/translation fault if the @ page table was modified by another CPU. enable_irq - tst r3, #PSR_T_BIT @ Thumb mode? - bne __und_usr_thumb - sub r4, r2, #4 @ ARM instr at LR - 4 -1: ldrt r0, [r4] - ARM_BE8(rev r0, r0) @ little endian instruction - - uaccess_disable ip - - @ r0 = 32-bit ARM instruction which caused the exception - @ r2 = PC value for the following instruction (:= regs->ARM_pc) - @ r4 = PC value for the faulting instruction - @ lr = 32-bit undefined instruction function - badr lr, __und_usr_fault_32 - b call_fpe - -__und_usr_thumb: - @ Thumb instruction - sub r4, r2, #2 @ First half of thumb instr at LR - 2 -#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7 -/* - * Thumb-2 instruction handling. Note that because pre-v6 and >= v6 platforms - * can never be supported in a single kernel, this code is not applicable at - * all when __LINUX_ARM_ARCH__ < 6. This allows simplifying assumptions to be - * made about .arch directives. - */ -#if __LINUX_ARM_ARCH__ < 7 -/* If the target CPU may not be Thumb-2-capable, a run-time check is needed: */ - ldr_va r5, cpu_architecture - cmp r5, #CPU_ARCH_ARMv7 - blo __und_usr_fault_16 @ 16bit undefined instruction -/* - * The following code won't get run unless the running CPU really is v7, so - * coding round the lack of ldrht on older arches is pointless. Temporarily - * override the assembler target arch with the minimum required instead: - */ - .arch armv6t2 + tst r5, #PSR_T_BIT @ Thumb mode? + mov r1, #2 @ set insn size to 2 for Thumb + bne 0f @ handle as Thumb undef exception +#ifdef CONFIG_FPE_NWFPE + adr r9, ret_from_exception + bl call_fpe @ returns via R9 on success #endif -2: ldrht r5, [r4] -ARM_BE8(rev16 r5, r5) @ little endian instruction - cmp r5, #0xe800 @ 32bit instruction if xx != 0 - blo __und_usr_fault_16_pan @ 16bit undefined instruction -3: ldrht r0, [r2] -ARM_BE8(rev16 r0, r0) @ little endian instruction + mov r1, #4 @ set insn size to 4 for ARM +0: mov r0, sp uaccess_disable ip - add r2, r2, #2 @ r2 is PC + 2, make it PC + 4 - str r2, [sp, #S_PC] @ it's a 2x16bit instr, update - orr r0, r0, r5, lsl #16 - badr lr, __und_usr_fault_32 - @ r0 = the two 16-bit Thumb instructions which caused the exception - @ r2 = PC value for the following Thumb instruction (:= regs->ARM_pc) - @ r4 = PC value for the first 16-bit Thumb instruction - @ lr = 32bit undefined instruction function - -#if __LINUX_ARM_ARCH__ < 7 -/* If the target arch was overridden, change it back: */ -#ifdef CONFIG_CPU_32v6K - .arch armv6k -#else - .arch armv6 -#endif -#endif /* __LINUX_ARM_ARCH__ < 7 */ -#else /* !(CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7) */ - b __und_usr_fault_16 -#endif + bl __und_fault + b ret_from_exception UNWIND(.fnend) ENDPROC(__und_usr) -/* - * The out of line fixup for the ldrt instructions above. - */ - .pushsection .text.fixup, "ax" - .align 2 -4: str r4, [sp, #S_PC] @ retry current instruction - ret r9 - .popsection - .pushsection __ex_table,"a" - .long 1b, 4b -#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7 - .long 2b, 4b - .long 3b, 4b -#endif - .popsection - -/* - * Check whether the instruction is a co-processor instruction. - * If yes, we need to call the relevant co-processor handler. - * - * Note that we don't do a full check here for the co-processor - * instructions; all instructions with bit 27 set are well - * defined. The only instructions that should fault are the - * co-processor instructions. However, we have to watch out - * for the ARM6/ARM7 SWI bug. - * - * NEON is a special case that has to be handled here. Not all - * NEON instructions are co-processor instructions, so we have - * to make a special case of checking for them. Plus, there's - * five groups of them, so we have a table of mask/opcode pairs - * to check against, and if any match then we branch off into the - * NEON handler code. - * - * Emulators may wish to make use of the following registers: - * r0 = instruction opcode (32-bit ARM or two 16-bit Thumb) - * r2 = PC value to resume execution after successful emulation - * r9 = normal "successful" return address - * r10 = this threads thread_info structure - * lr = unrecognised instruction return address - * IRQs enabled, FIQs enabled. - */ - @ - @ Fall-through from Thumb-2 __und_usr - @ -#ifdef CONFIG_NEON - get_thread_info r10 @ get current thread - adr r6, .LCneon_thumb_opcodes - b 2f -#endif -call_fpe: - get_thread_info r10 @ get current thread -#ifdef CONFIG_NEON - adr r6, .LCneon_arm_opcodes -2: ldr r5, [r6], #4 @ mask value - ldr r7, [r6], #4 @ opcode bits matching in mask - cmp r5, #0 @ end mask? - beq 1f - and r8, r0, r5 - cmp r8, r7 @ NEON instruction? - bne 2b - mov r7, #1 - strb r7, [r10, #TI_USED_CP + 10] @ mark CP#10 as used - strb r7, [r10, #TI_USED_CP + 11] @ mark CP#11 as used - b do_vfp @ let VFP handler handle this -1: -#endif - tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27 - tstne r0, #0x04000000 @ bit 26 set on both ARM and Thumb-2 - reteq lr - and r8, r0, #0x00000f00 @ mask out CP number - mov r7, #1 - add r6, r10, r8, lsr #8 @ add used_cp[] array offset first - strb r7, [r6, #TI_USED_CP] @ set appropriate used_cp[] -#ifdef CONFIG_IWMMXT - @ Test if we need to give access to iWMMXt coprocessors - ldr r5, [r10, #TI_FLAGS] - rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only - movscs r7, r5, lsr #(TIF_USING_IWMMXT + 1) - bcs iwmmxt_task_enable -#endif - ARM( add pc, pc, r8, lsr #6 ) - THUMB( lsr r8, r8, #6 ) - THUMB( add pc, r8 ) - nop - - ret.w lr @ CP#0 - W(b) do_fpe @ CP#1 (FPE) - W(b) do_fpe @ CP#2 (FPE) - ret.w lr @ CP#3 - ret.w lr @ CP#4 - ret.w lr @ CP#5 - ret.w lr @ CP#6 - ret.w lr @ CP#7 - ret.w lr @ CP#8 - ret.w lr @ CP#9 -#ifdef CONFIG_VFP - W(b) do_vfp @ CP#10 (VFP) - W(b) do_vfp @ CP#11 (VFP) -#else - ret.w lr @ CP#10 (VFP) - ret.w lr @ CP#11 (VFP) -#endif - ret.w lr @ CP#12 - ret.w lr @ CP#13 - ret.w lr @ CP#14 (Debug) - ret.w lr @ CP#15 (Control) - -#ifdef CONFIG_NEON - .align 6 - -.LCneon_arm_opcodes: - .word 0xfe000000 @ mask - .word 0xf2000000 @ opcode - - .word 0xff100000 @ mask - .word 0xf4000000 @ opcode - - .word 0x00000000 @ mask - .word 0x00000000 @ opcode - -.LCneon_thumb_opcodes: - .word 0xef000000 @ mask - .word 0xef000000 @ opcode - - .word 0xff100000 @ mask - .word 0xf9000000 @ opcode - - .word 0x00000000 @ mask - .word 0x00000000 @ opcode -#endif - -do_fpe: - add r10, r10, #TI_FPSTATE @ r10 = workspace - ldr_va pc, fp_enter, tmp=r4 @ Call FP module USR entry point - -/* - * The FP module is called with these registers set: - * r0 = instruction - * r2 = PC+4 - * r9 = normal "successful" return address - * r10 = FP workspace - * lr = unrecognised FP instruction return address - */ - - .pushsection .data - .align 2 -ENTRY(fp_enter) - .word no_fp - .popsection - -ENTRY(no_fp) - ret lr -ENDPROC(no_fp) - -__und_usr_fault_32: - mov r1, #4 - b 1f -__und_usr_fault_16_pan: - uaccess_disable ip -__und_usr_fault_16: - mov r1, #2 -1: mov r0, sp - badr lr, ret_from_exception - b __und_fault -ENDPROC(__und_usr_fault_32) -ENDPROC(__und_usr_fault_16) - .align 5 __pabt_usr: usr_entry @@ -787,6 +562,13 @@ ENTRY(__switch_to) @ entries covering the vmalloc region. @ ldr r2, [ip] +#ifdef CONFIG_KASAN_VMALLOC + @ Also dummy read from the KASAN shadow memory for the new stack if we + @ are using KASAN + mov_l r2, KASAN_SHADOW_OFFSET + add r2, r2, ip, lsr #KASAN_SHADOW_SCALE_SHIFT + ldr r2, [r2] +#endif #endif @ When CONFIG_THREAD_INFO_IN_TASK=n, the update of SP itself is what @@ -875,7 +657,7 @@ ENDPROC(__bad_stack) * existing ones. This mechanism should be used only for things that are * really small and justified, and not be abused freely. * - * See Documentation/arm/kernel_user_helpers.rst for formal definitions. + * See Documentation/arch/arm/kernel_user_helpers.rst for formal definitions. */ THUMB( .arm ) @@ -1297,6 +1079,7 @@ vector_addrexcptn: .globl vector_fiq .section .vectors, "ax", %progbits + RELOC_TEXT_NONE W(b) vector_rst W(b) vector_und ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_swi ) @@ -1310,6 +1093,7 @@ THUMB( .reloc ., R_ARM_THM_PC12, .L__vector_swi ) #ifdef CONFIG_HARDEN_BRANCH_HISTORY .section .vectors.bhb.loop8, "ax", %progbits + RELOC_TEXT_NONE W(b) vector_rst W(b) vector_bhb_loop8_und ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_bhb_loop8_swi ) @@ -1322,6 +1106,7 @@ THUMB( .reloc ., R_ARM_THM_PC12, .L__vector_bhb_loop8_swi ) W(b) vector_bhb_loop8_fiq .section .vectors.bhb.bpiall, "ax", %progbits + RELOC_TEXT_NONE W(b) vector_rst W(b) vector_bhb_bpiall_und ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_bhb_bpiall_swi ) diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 405a607b754f..88336a1292bb 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -9,22 +9,13 @@ #include <asm/unistd.h> #include <asm/ftrace.h> #include <asm/unwind.h> -#include <asm/memory.h> +#include <asm/page.h> #ifdef CONFIG_AEABI #include <asm/unistd-oabi.h> #endif .equ NR_syscalls, __NR_syscalls - .macro arch_ret_to_user, tmp -#ifdef CONFIG_ARCH_IOP32X - mrc p15, 0, \tmp, c15, c1, 0 - tst \tmp, #(1 << 6) - bicne \tmp, \tmp, #(1 << 6) - mcrne p15, 0, \tmp, c15, c1, 0 @ Disable cp6 access -#endif - .endm - #include "entry-header.S" saved_psr .req r8 @@ -55,10 +46,6 @@ __ret_fast_syscall: movs r1, r1, lsl #16 bne fast_work_pending - - /* perform architecture specific actions before user return */ - arch_ret_to_user r1 - restore_user_regs fast = 1, offset = S_OFF UNWIND(.fnend ) ENDPROC(ret_fast_syscall) @@ -103,6 +90,7 @@ slow_work_pending: cmp r0, #0 beq no_work_pending movlt scno, #(__NR_restart_syscall - __NR_SYSCALL_BASE) + str scno, [tsk, #TI_ABI_SYSCALL] @ make sure tracers see update ldmia sp, {r0 - r6} @ have to reload r0 - r6 b local_restart @ ... and off we go ENDPROC(ret_fast_syscall) @@ -129,10 +117,11 @@ ENTRY(ret_to_user_from_irq) no_work_pending: asm_trace_hardirqs_on save = 0 - /* perform architecture specific actions before user return */ - arch_ret_to_user r1 ct_user_enter save = 0 +#ifdef CONFIG_KSTACK_ERASE + bl stackleak_erase_on_task_stack +#endif restore_user_regs fast = 0, offset = 0 ENDPROC(ret_to_user_from_irq) ENDPROC(ret_to_user) diff --git a/arch/arm/kernel/entry-ftrace.S b/arch/arm/kernel/entry-ftrace.S index 3e7bcaca5e07..e24ee559af81 100644 --- a/arch/arm/kernel/entry-ftrace.S +++ b/arch/arm/kernel/entry-ftrace.S @@ -257,11 +257,21 @@ ENDPROC(ftrace_graph_regs_caller) #ifdef CONFIG_FUNCTION_GRAPH_TRACER ENTRY(return_to_handler) - stmdb sp!, {r0-r3} - add r0, sp, #16 @ sp at exit of instrumented routine + mov ip, sp @ sp at exit of instrumented routine + sub sp, #PT_REGS_SIZE + str r0, [sp, #S_R0] + str r1, [sp, #S_R1] + str r2, [sp, #S_R2] + str r3, [sp, #S_R3] + str ip, [sp, #S_FP] + mov r0, sp bl ftrace_return_to_handler - mov lr, r0 @ r0 has real ret addr - ldmia sp!, {r0-r3} + mov lr, r0 @ r0 has real ret addr + ldr r3, [sp, #S_R3] + ldr r2, [sp, #S_R2] + ldr r1, [sp, #S_R1] + ldr r0, [sp, #S_R0] + add sp, sp, #PT_REGS_SIZE @ restore stack pointer ret lr ENDPROC(return_to_handler) #endif @@ -271,6 +281,10 @@ ENTRY(ftrace_stub) ret lr ENDPROC(ftrace_stub) +ENTRY(ftrace_stub_graph) + ret lr +ENDPROC(ftrace_stub_graph) + #ifdef CONFIG_DYNAMIC_FTRACE __INIT diff --git a/arch/arm/kernel/entry-v7m.S b/arch/arm/kernel/entry-v7m.S index de8a60363c85..52bacf07ba16 100644 --- a/arch/arm/kernel/entry-v7m.S +++ b/arch/arm/kernel/entry-v7m.S @@ -6,7 +6,7 @@ * * Low-level vector interface routines for the ARMv7-M architecture */ -#include <asm/memory.h> +#include <asm/page.h> #include <asm/glue.h> #include <asm/thread_notify.h> #include <asm/v7m.h> diff --git a/arch/arm/kernel/fiq.c b/arch/arm/kernel/fiq.c index 98ca3e3fa847..d2c8e5313539 100644 --- a/arch/arm/kernel/fiq.c +++ b/arch/arm/kernel/fiq.c @@ -45,6 +45,7 @@ #include <asm/cacheflush.h> #include <asm/cp15.h> #include <asm/fiq.h> +#include <asm/mach/irq.h> #include <asm/irq.h> #include <asm/traps.h> diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c index a0b6d1e3812f..845acf9ce21e 100644 --- a/arch/arm/kernel/ftrace.c +++ b/arch/arm/kernel/ftrace.c @@ -23,7 +23,7 @@ #include <asm/insn.h> #include <asm/set_memory.h> #include <asm/stacktrace.h> -#include <asm/patch.h> +#include <asm/text-patching.h> /* * The compiler emitted profiling hook consists of @@ -232,11 +232,24 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, unsigned long old; if (unlikely(atomic_read(¤t->tracing_graph_pause))) +err_out: return; if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) { - /* FP points one word below parent's top of stack */ - frame_pointer += 4; + /* + * Usually, the stack frames are contiguous in memory but cases + * have been observed where the next stack frame does not live + * at 'frame_pointer + 4' as this code used to assume. + * + * Instead, dereference the field in the stack frame that + * stores the SP of the calling frame: to avoid unbounded + * recursion, this cannot involve any ftrace instrumented + * functions, so use the __get_kernel_nofault() primitive + * directly. + */ + __get_kernel_nofault(&frame_pointer, + (unsigned long *)(frame_pointer - 8), + unsigned long, err_out); } else { struct stackframe frame = { .fp = frame_pointer, diff --git a/arch/arm/kernel/head-inflate-data.c b/arch/arm/kernel/head-inflate-data.c index 89a52104d32a..225c0699a12c 100644 --- a/arch/arm/kernel/head-inflate-data.c +++ b/arch/arm/kernel/head-inflate-data.c @@ -8,16 +8,13 @@ #include <linux/init.h> #include <linux/zutil.h> +#include "head.h" /* for struct inflate_state */ #include "../../../lib/zlib_inflate/inftrees.h" #include "../../../lib/zlib_inflate/inflate.h" #include "../../../lib/zlib_inflate/infutil.h" -extern char __data_loc[]; -extern char _edata_loc[]; -extern char _sdata[]; - /* * This code is called very early during the boot process to decompress * the .data segment stored compressed in ROM. Therefore none of the global diff --git a/arch/arm/kernel/head-nommu.S b/arch/arm/kernel/head-nommu.S index 950bef83339f..b9d6818f1ee1 100644 --- a/arch/arm/kernel/head-nommu.S +++ b/arch/arm/kernel/head-nommu.S @@ -14,12 +14,11 @@ #include <asm/assembler.h> #include <asm/ptrace.h> #include <asm/asm-offsets.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/cp15.h> #include <asm/thread_info.h> #include <asm/v7m.h> #include <asm/mpu.h> -#include <asm/page.h> /* * Kernel startup entry point. diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index 29e2900178a1..f22c50d4bd41 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -17,7 +17,7 @@ #include <asm/domain.h> #include <asm/ptrace.h> #include <asm/asm-offsets.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/thread_info.h> #if defined(CONFIG_DEBUG_LL) && !defined(CONFIG_DEBUG_SEMIHOSTING) @@ -252,18 +252,23 @@ __create_page_tables: */ add r0, r4, #KERNEL_OFFSET >> (SECTION_SHIFT - PMD_ENTRY_ORDER) ldr r6, =(_end - 1) + + /* For XIP, kernel_sec_start/kernel_sec_end are currently in RO memory */ +#ifndef CONFIG_XIP_KERNEL adr_l r5, kernel_sec_start @ _pa(kernel_sec_start) #if defined CONFIG_CPU_ENDIAN_BE8 || defined CONFIG_CPU_ENDIAN_BE32 str r8, [r5, #4] @ Save physical start of kernel (BE) #else str r8, [r5] @ Save physical start of kernel (LE) #endif +#endif orr r3, r8, r7 @ Add the MMU flags add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ENTRY_ORDER) 1: str r3, [r0], #1 << PMD_ENTRY_ORDER add r3, r3, #1 << SECTION_SHIFT cmp r0, r6 bls 1b +#ifndef CONFIG_XIP_KERNEL eor r3, r3, r7 @ Remove the MMU flags adr_l r5, kernel_sec_end @ _pa(kernel_sec_end) #if defined CONFIG_CPU_ENDIAN_BE8 || defined CONFIG_CPU_ENDIAN_BE32 @@ -271,8 +276,7 @@ __create_page_tables: #else str r3, [r5] @ Save physical end of kernel (LE) #endif - -#ifdef CONFIG_XIP_KERNEL +#else /* * Map the kernel image separately as it is not located in RAM. */ @@ -344,7 +348,7 @@ __create_page_tables: ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags #endif -#if defined(CONFIG_ARCH_NETWINDER) || defined(CONFIG_ARCH_CATS) +#if defined(CONFIG_ARCH_NETWINDER) /* * If we're using the NetWinder or CATS, we also need to map * in the 16550-type serial port for the debug messages @@ -407,7 +411,11 @@ ENTRY(secondary_startup) /* * Use the page tables supplied from __cpu_up. */ +#ifdef CONFIG_XIP_KERNEL + ldr r3, =(secondary_data + PLAT_PHYS_OFFSET - PAGE_OFFSET) +#else adr_l r3, secondary_data +#endif mov_l r12, __secondary_switched ldrd r4, r5, [r3, #0] @ get secondary_data.pgdir ARM_BE8(eor r4, r4, r5) @ Swap r5 and r4 in BE: diff --git a/arch/arm/kernel/head.h b/arch/arm/kernel/head.h new file mode 100644 index 000000000000..0eb5accf7141 --- /dev/null +++ b/arch/arm/kernel/head.h @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-only + +extern char __data_loc[]; +extern char _edata_loc[]; +extern char _sdata[]; + +int __init __inflate_kernel_data(void); diff --git a/arch/arm/kernel/hibernate.c b/arch/arm/kernel/hibernate.c index 2373020af965..38a90a3d12b2 100644 --- a/arch/arm/kernel/hibernate.c +++ b/arch/arm/kernel/hibernate.c @@ -19,7 +19,7 @@ #include <asm/system_misc.h> #include <asm/idmap.h> #include <asm/suspend.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/sections.h> #include "reboot.h" diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index 054e9199f30d..cd4b34c96e35 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -17,6 +17,7 @@ #include <linux/perf_event.h> #include <linux/hw_breakpoint.h> #include <linux/smp.h> +#include <linux/cfi.h> #include <linux/cpu_pm.h> #include <linux/coresight.h> @@ -903,6 +904,37 @@ unlock: watchpoint_single_step_handler(addr); } +#ifdef CONFIG_CFI +static void hw_breakpoint_cfi_handler(struct pt_regs *regs) +{ + /* + * TODO: implementing target and type to pass to CFI using the more + * elaborate report_cfi_failure() requires compiler work. To be able + * to properly extract target information the compiler needs to + * emit a stable instructions sequence for the CFI checks so we can + * decode the instructions preceding the trap and figure out which + * registers were used. + */ + + switch (report_cfi_failure_noaddr(regs, instruction_pointer(regs))) { + case BUG_TRAP_TYPE_BUG: + die("Oops - CFI", regs, 0); + break; + case BUG_TRAP_TYPE_WARN: + /* Skip the breaking instruction */ + instruction_pointer(regs) += 4; + break; + default: + die("Unknown CFI error", regs, 0); + break; + } +} +#else +static void hw_breakpoint_cfi_handler(struct pt_regs *regs) +{ +} +#endif + /* * Called from either the Data Abort Handler [watchpoint] or the * Prefetch Abort Handler [breakpoint] with interrupts disabled. @@ -932,6 +964,9 @@ static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr, case ARM_ENTRY_SYNC_WATCHPOINT: watchpoint_handler(addr, fsr, regs); break; + case ARM_ENTRY_CFI_BREAKPOINT: + hw_breakpoint_cfi_handler(regs); + break; default: ret = 1; /* Unhandled fault. */ } diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index fe28fc1f759d..e1993e28a9ec 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -32,6 +32,7 @@ #include <linux/kallsyms.h> #include <linux/proc_fs.h> #include <linux/export.h> +#include <linux/vmalloc.h> #include <asm/hardware/cache-l2x0.h> #include <asm/hardware/cache-uniphier.h> @@ -110,7 +111,7 @@ void handle_IRQ(unsigned int irq, struct pt_regs *regs) * Some hardware gives randomly wrong interrupts. Rather * than crashing, do something sensible. */ - if (unlikely(!irq || irq >= nr_irqs)) + if (unlikely(!irq || irq >= irq_get_nr_irqs())) desc = NULL; else desc = irq_to_desc(irq); @@ -150,7 +151,6 @@ void __init init_IRQ(void) #ifdef CONFIG_SPARSE_IRQ int __init arch_probe_nr_irqs(void) { - nr_irqs = machine_desc->nr_irqs ? machine_desc->nr_irqs : NR_IRQS; - return nr_irqs; + return irq_set_nr_irqs(machine_desc->nr_irqs ? : NR_IRQS); } #endif diff --git a/arch/arm/kernel/isa.c b/arch/arm/kernel/isa.c index d8a509c5d5bd..db8be609fab2 100644 --- a/arch/arm/kernel/isa.c +++ b/arch/arm/kernel/isa.c @@ -16,7 +16,7 @@ static unsigned int isa_membase, isa_portbase, isa_portshift; -static struct ctl_table ctl_isa_vars[4] = { +static const struct ctl_table ctl_isa_vars[] = { { .procname = "membase", .data = &isa_membase, @@ -35,32 +35,16 @@ static struct ctl_table ctl_isa_vars[4] = { .maxlen = sizeof(isa_portshift), .mode = 0444, .proc_handler = proc_dointvec, - }, {} + }, }; static struct ctl_table_header *isa_sysctl_header; -static struct ctl_table ctl_isa[2] = { - { - .procname = "isa", - .mode = 0555, - .child = ctl_isa_vars, - }, {} -}; - -static struct ctl_table ctl_bus[2] = { - { - .procname = "bus", - .mode = 0555, - .child = ctl_isa, - }, {} -}; - void __init register_isa_ports(unsigned int membase, unsigned int portbase, unsigned int portshift) { isa_membase = membase; isa_portbase = portbase; isa_portshift = portshift; - isa_sysctl_header = register_sysctl_table(ctl_bus); + isa_sysctl_header = register_sysctl("bus/isa", ctl_isa_vars); } diff --git a/arch/arm/kernel/iwmmxt.S b/arch/arm/kernel/iwmmxt.S index d2b4ac06e4ed..4a335d3c5969 100644 --- a/arch/arm/kernel/iwmmxt.S +++ b/arch/arm/kernel/iwmmxt.S @@ -18,18 +18,6 @@ #include <asm/assembler.h> #include "iwmmxt.h" -#if defined(CONFIG_CPU_PJ4) || defined(CONFIG_CPU_PJ4B) -#define PJ4(code...) code -#define XSC(code...) -#elif defined(CONFIG_CPU_MOHAWK) || \ - defined(CONFIG_CPU_XSC3) || \ - defined(CONFIG_CPU_XSCALE) -#define PJ4(code...) -#define XSC(code...) code -#else -#error "Unsupported iWMMXt architecture" -#endif - #define MMX_WR0 (0x00) #define MMX_WR1 (0x08) #define MMX_WR2 (0x10) @@ -58,9 +46,19 @@ .text .arm +ENTRY(iwmmxt_undef_handler) + push {r9, r10, lr} + get_thread_info r10 + mov r9, pc + b iwmmxt_task_enable + mov r0, #0 + pop {r9, r10, pc} +ENDPROC(iwmmxt_undef_handler) + /* * Lazy switching of Concan coprocessor context * + * r0 = struct pt_regs pointer * r10 = struct thread_info pointer * r9 = ret_from_exception * lr = undefined instr exit @@ -71,25 +69,21 @@ ENTRY(iwmmxt_task_enable) inc_preempt_count r10, r3 - XSC(mrc p15, 0, r2, c15, c1, 0) - PJ4(mrc p15, 0, r2, c1, c0, 2) + mrc p15, 0, r2, c15, c1, 0 @ CP0 and CP1 accessible? - XSC(tst r2, #0x3) - PJ4(tst r2, #0xf) + tst r2, #0x3 bne 4f @ if so no business here @ enable access to CP0 and CP1 - XSC(orr r2, r2, #0x3) - XSC(mcr p15, 0, r2, c15, c1, 0) - PJ4(orr r2, r2, #0xf) - PJ4(mcr p15, 0, r2, c1, c0, 2) + orr r2, r2, #0x3 + mcr p15, 0, r2, c15, c1, 0 ldr r3, =concan_owner - add r0, r10, #TI_IWMMXT_STATE @ get task Concan save area - ldr r2, [sp, #60] @ current task pc value + ldr r2, [r0, #S_PC] @ current task pc value ldr r1, [r3] @ get current Concan owner - str r0, [r3] @ this task now owns Concan regs sub r2, r2, #4 @ adjust pc back - str r2, [sp, #60] + str r2, [r0, #S_PC] + add r0, r10, #TI_IWMMXT_STATE @ get task Concan save area + str r0, [r3] @ this task now owns Concan regs mrc p15, 0, r2, c2, c0, 0 mov r2, r2 @ cpwait @@ -208,12 +202,9 @@ ENTRY(iwmmxt_task_disable) bne 1f @ no: quit @ enable access to CP0 and CP1 - XSC(mrc p15, 0, r4, c15, c1, 0) - XSC(orr r4, r4, #0x3) - XSC(mcr p15, 0, r4, c15, c1, 0) - PJ4(mrc p15, 0, r4, c1, c0, 2) - PJ4(orr r4, r4, #0xf) - PJ4(mcr p15, 0, r4, c1, c0, 2) + mrc p15, 0, r4, c15, c1, 0 + orr r4, r4, #0x3 + mcr p15, 0, r4, c15, c1, 0 mov r0, #0 @ nothing to load str r0, [r3] @ no more current owner @@ -222,10 +213,8 @@ ENTRY(iwmmxt_task_disable) bl concan_save @ disable access to CP0 and CP1 - XSC(bic r4, r4, #0x3) - XSC(mcr p15, 0, r4, c15, c1, 0) - PJ4(bic r4, r4, #0xf) - PJ4(mcr p15, 0, r4, c1, c0, 2) + bic r4, r4, #0x3 + mcr p15, 0, r4, c15, c1, 0 mrc p15, 0, r2, c2, c0, 0 mov r2, r2 @ cpwait @@ -320,11 +309,9 @@ ENDPROC(iwmmxt_task_restore) */ ENTRY(iwmmxt_task_switch) - XSC(mrc p15, 0, r1, c15, c1, 0) - PJ4(mrc p15, 0, r1, c1, c0, 2) + mrc p15, 0, r1, c15, c1, 0 @ CP0 and CP1 accessible? - XSC(tst r1, #0x3) - PJ4(tst r1, #0xf) + tst r1, #0x3 bne 1f @ yes: block them for next task ldr r2, =concan_owner @@ -334,10 +321,8 @@ ENTRY(iwmmxt_task_switch) retne lr @ no: leave Concan disabled 1: @ flip Concan access - XSC(eor r1, r1, #0x3) - XSC(mcr p15, 0, r1, c15, c1, 0) - PJ4(eor r1, r1, #0xf) - PJ4(mcr p15, 0, r1, c1, c0, 2) + eor r1, r1, #0x3 + mcr p15, 0, r1, c15, c1, 0 mrc p15, 0, r1, c2, c0, 0 sub pc, lr, r1, lsr #32 @ cpwait and return diff --git a/arch/arm/kernel/jump_label.c b/arch/arm/kernel/jump_label.c index eb9c24b6e8e2..a06a92d0f550 100644 --- a/arch/arm/kernel/jump_label.c +++ b/arch/arm/kernel/jump_label.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/kernel.h> #include <linux/jump_label.h> -#include <asm/patch.h> +#include <asm/text-patching.h> #include <asm/insn.h> static void __arch_jump_label_transform(struct jump_entry *entry, diff --git a/arch/arm/kernel/kgdb.c b/arch/arm/kernel/kgdb.c index 22f937e6f3ff..ab76c55fd610 100644 --- a/arch/arm/kernel/kgdb.c +++ b/arch/arm/kernel/kgdb.c @@ -15,7 +15,7 @@ #include <linux/kgdb.h> #include <linux/uaccess.h> -#include <asm/patch.h> +#include <asm/text-patching.h> #include <asm/traps.h> struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = diff --git a/arch/arm/kernel/machine_kexec.c b/arch/arm/kernel/machine_kexec.c index 46364b699cc3..dd430477e7c1 100644 --- a/arch/arm/kernel/machine_kexec.c +++ b/arch/arm/kernel/machine_kexec.c @@ -94,16 +94,28 @@ static void machine_crash_nonpanic_core(void *unused) } } +static DEFINE_PER_CPU(call_single_data_t, cpu_stop_csd) = + CSD_INIT(machine_crash_nonpanic_core, NULL); + void crash_smp_send_stop(void) { static int cpus_stopped; unsigned long msecs; + call_single_data_t *csd; + int cpu, this_cpu = raw_smp_processor_id(); if (cpus_stopped) return; atomic_set(&waiting_for_crash_ipi, num_online_cpus() - 1); - smp_call_function(machine_crash_nonpanic_core, NULL, false); + for_each_online_cpu(cpu) { + if (cpu == this_cpu) + continue; + + csd = &per_cpu(cpu_stop_csd, cpu); + smp_call_function_single_async(cpu, csd); + } + msecs = 1000; /* Wait at most a second for the other cpus to stop */ while ((atomic_read(&waiting_for_crash_ipi) > 0) && msecs) { mdelay(1); @@ -115,29 +127,6 @@ void crash_smp_send_stop(void) cpus_stopped = 1; } -static void machine_kexec_mask_interrupts(void) -{ - unsigned int i; - struct irq_desc *desc; - - for_each_irq_desc(i, desc) { - struct irq_chip *chip; - - chip = irq_desc_get_chip(desc); - if (!chip) - continue; - - if (chip->irq_eoi && irqd_irq_inprogress(&desc->irq_data)) - chip->irq_eoi(&desc->irq_data); - - if (chip->irq_mask) - chip->irq_mask(&desc->irq_data); - - if (chip->irq_disable && !irqd_irq_disabled(&desc->irq_data)) - chip->irq_disable(&desc->irq_data); - } -} - void machine_crash_shutdown(struct pt_regs *regs) { local_irq_disable(); @@ -186,10 +175,3 @@ void machine_kexec(struct kimage *image) soft_restart(reboot_entry_phys); } - -void arch_crash_save_vmcoreinfo(void) -{ -#ifdef CONFIG_ARM_LPAE - VMCOREINFO_CONFIG(ARM_LPAE); -#endif -} diff --git a/arch/arm/kernel/module-plts.c b/arch/arm/kernel/module-plts.c index af7c322ebed6..354ce16d83cb 100644 --- a/arch/arm/kernel/module-plts.c +++ b/arch/arm/kernel/module-plts.c @@ -28,11 +28,6 @@ static const u32 fixed_plts[] = { #endif }; -static bool in_init(const struct module *mod, unsigned long loc) -{ - return loc - (u32)mod->init_layout.base < mod->init_layout.size; -} - static void prealloc_fixed(struct mod_plt_sec *pltsec, struct plt_entries *plt) { int i; @@ -50,8 +45,8 @@ static void prealloc_fixed(struct mod_plt_sec *pltsec, struct plt_entries *plt) u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val) { - struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core : - &mod->arch.init; + struct mod_plt_sec *pltsec = !within_module_init(loc, mod) ? + &mod->arch.core : &mod->arch.init; struct plt_entries *plt; int idx; @@ -256,7 +251,7 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, /* sort by type and symbol index */ sort(rels, numrels, sizeof(Elf32_Rel), cmp_rel, NULL); - if (strncmp(secstrings + dstsec->sh_name, ".init", 5) != 0) + if (!module_init_layout_section(secstrings + dstsec->sh_name)) core_plts += count_plts(syms, dstsec->sh_addr, rels, numrels, s->sh_info); else @@ -290,11 +285,9 @@ bool in_module_plt(unsigned long loc) struct module *mod; bool ret; - preempt_disable(); + guard(rcu)(); mod = __module_text_address(loc); ret = mod && (loc - (u32)mod->arch.core.plt_ent < mod->arch.core.plt_count * PLT_ENT_SIZE || loc - (u32)mod->arch.init.plt_ent < mod->arch.init.plt_count * PLT_ENT_SIZE); - preempt_enable(); - return ret; } diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index d59c36dc0494..55ca3fcd37e8 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -12,48 +12,14 @@ #include <linux/kernel.h> #include <linux/mm.h> #include <linux/elf.h> -#include <linux/vmalloc.h> #include <linux/fs.h> #include <linux/string.h> -#include <linux/gfp.h> #include <asm/sections.h> #include <asm/smp_plat.h> #include <asm/unwind.h> #include <asm/opcodes.h> -#ifdef CONFIG_XIP_KERNEL -/* - * The XIP kernel text is mapped in the module area for modules and - * some other stuff to work without any indirect relocations. - * MODULES_VADDR is redefined here and not in asm/memory.h to avoid - * recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off. - */ -#undef MODULES_VADDR -#define MODULES_VADDR (((unsigned long)_exiprom + ~PMD_MASK) & PMD_MASK) -#endif - -#ifdef CONFIG_MMU -void *module_alloc(unsigned long size) -{ - gfp_t gfp_mask = GFP_KERNEL; - void *p; - - /* Silence the initial allocation */ - if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS)) - gfp_mask |= __GFP_NOWARN; - - p = __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - gfp_mask, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, - __builtin_return_address(0)); - if (!IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || p) - return p; - return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, - GFP_KERNEL, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, - __builtin_return_address(0)); -} -#endif - bool module_init_section(const char *name) { return strstarts(name, ".init") || @@ -169,8 +135,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, offset = __mem_to_opcode_arm(*(u32 *)loc); offset = (offset & 0x00ffffff) << 2; - if (offset & 0x02000000) - offset -= 0x04000000; + offset = sign_extend32(offset, 25); offset += sym->st_value - loc; @@ -236,7 +201,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, case R_ARM_MOVT_PREL: offset = tmp = __mem_to_opcode_arm(*(u32 *)loc); offset = ((offset & 0xf0000) >> 4) | (offset & 0xfff); - offset = (offset ^ 0x8000) - 0x8000; + offset = sign_extend32(offset, 15); offset += sym->st_value; if (ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_PREL || @@ -344,8 +309,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, ((~(j2 ^ sign) & 1) << 22) | ((upper & 0x03ff) << 12) | ((lower & 0x07ff) << 1); - if (offset & 0x01000000) - offset -= 0x02000000; + offset = sign_extend32(offset, 24); offset += sym->st_value - loc; /* @@ -401,7 +365,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, offset = ((upper & 0x000f) << 12) | ((upper & 0x0400) << 1) | ((lower & 0x7000) >> 4) | (lower & 0x00ff); - offset = (offset ^ 0x8000) - 0x8000; + offset = sign_extend32(offset, 15); offset += sym->st_value; if (ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_PREL || @@ -431,11 +395,6 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, return 0; } -struct mod_unwind_map { - const Elf_Shdr *unw_sec; - const Elf_Shdr *txt_sec; -}; - static const Elf_Shdr *find_mod_section(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs, const char *name) { @@ -525,7 +484,7 @@ module_arch_cleanup(struct module *mod) #endif } -void __weak module_arch_freeing_init(struct module *mod) +void module_arch_freeing_init(struct module *mod) { #ifdef CONFIG_ARM_UNWIND struct unwind_table *init = mod->arch.init_table; diff --git a/arch/arm/kernel/patch.c b/arch/arm/kernel/patch.c index e9e828b6bb30..4d45e60cd46d 100644 --- a/arch/arm/kernel/patch.c +++ b/arch/arm/kernel/patch.c @@ -9,7 +9,7 @@ #include <asm/fixmap.h> #include <asm/smp_plat.h> #include <asm/opcodes.h> -#include <asm/patch.h> +#include <asm/text-patching.h> struct patch { void *addr; diff --git a/arch/arm/kernel/perf_callchain.c b/arch/arm/kernel/perf_callchain.c index 7147edbe56c6..a2601b1ef318 100644 --- a/arch/arm/kernel/perf_callchain.c +++ b/arch/arm/kernel/perf_callchain.c @@ -85,8 +85,7 @@ static bool callchain_trace(void *data, unsigned long pc) { struct perf_callchain_entry_ctx *entry = data; - perf_callchain_store(entry, pc); - return true; + return perf_callchain_store(entry, pc) == 0; } void @@ -97,20 +96,3 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *re arm_get_current_stackframe(regs, &fr); walk_stackframe(&fr, callchain_trace, entry); } - -unsigned long perf_instruction_pointer(struct pt_regs *regs) -{ - return instruction_pointer(regs); -} - -unsigned long perf_misc_flags(struct pt_regs *regs) -{ - int misc = 0; - - if (user_mode(regs)) - misc |= PERF_RECORD_MISC_USER; - else - misc |= PERF_RECORD_MISC_KERNEL; - - return misc; -} diff --git a/arch/arm/kernel/perf_event_v6.c b/arch/arm/kernel/perf_event_v6.c deleted file mode 100644 index 1ae99deeec54..000000000000 --- a/arch/arm/kernel/perf_event_v6.c +++ /dev/null @@ -1,590 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * ARMv6 Performance counter handling code. - * - * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles - * - * ARMv6 has 2 configurable performance counters and a single cycle counter. - * They all share a single reset bit but can be written to zero so we can use - * that for a reset. - * - * The counters can't be individually enabled or disabled so when we remove - * one event and replace it with another we could get spurious counts from the - * wrong event. However, we can take advantage of the fact that the - * performance counters can export events to the event bus, and the event bus - * itself can be monitored. This requires that we *don't* export the events to - * the event bus. The procedure for disabling a configurable counter is: - * - change the counter to count the ETMEXTOUT[0] signal (0x20). This - * effectively stops the counter from counting. - * - disable the counter's interrupt generation (each counter has it's - * own interrupt enable bit). - * Once stopped, the counter value can be written as 0 to reset. - * - * To enable a counter: - * - enable the counter's interrupt generation. - * - set the new event type. - * - * Note: the dedicated cycle counter only counts cycles and can't be - * enabled/disabled independently of the others. When we want to disable the - * cycle counter, we have to just disable the interrupt reporting and start - * ignoring that counter. When re-enabling, we have to reset the value and - * enable the interrupt. - */ - -#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K) - -#include <asm/cputype.h> -#include <asm/irq_regs.h> - -#include <linux/of.h> -#include <linux/perf/arm_pmu.h> -#include <linux/platform_device.h> - -enum armv6_perf_types { - ARMV6_PERFCTR_ICACHE_MISS = 0x0, - ARMV6_PERFCTR_IBUF_STALL = 0x1, - ARMV6_PERFCTR_DDEP_STALL = 0x2, - ARMV6_PERFCTR_ITLB_MISS = 0x3, - ARMV6_PERFCTR_DTLB_MISS = 0x4, - ARMV6_PERFCTR_BR_EXEC = 0x5, - ARMV6_PERFCTR_BR_MISPREDICT = 0x6, - ARMV6_PERFCTR_INSTR_EXEC = 0x7, - ARMV6_PERFCTR_DCACHE_HIT = 0x9, - ARMV6_PERFCTR_DCACHE_ACCESS = 0xA, - ARMV6_PERFCTR_DCACHE_MISS = 0xB, - ARMV6_PERFCTR_DCACHE_WBACK = 0xC, - ARMV6_PERFCTR_SW_PC_CHANGE = 0xD, - ARMV6_PERFCTR_MAIN_TLB_MISS = 0xF, - ARMV6_PERFCTR_EXPL_D_ACCESS = 0x10, - ARMV6_PERFCTR_LSU_FULL_STALL = 0x11, - ARMV6_PERFCTR_WBUF_DRAINED = 0x12, - ARMV6_PERFCTR_CPU_CYCLES = 0xFF, - ARMV6_PERFCTR_NOP = 0x20, -}; - -enum armv6_counters { - ARMV6_CYCLE_COUNTER = 0, - ARMV6_COUNTER0, - ARMV6_COUNTER1, -}; - -/* - * The hardware events that we support. We do support cache operations but - * we have harvard caches and no way to combine instruction and data - * accesses/misses in hardware. - */ -static const unsigned armv6_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV6_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV6_PERFCTR_INSTR_EXEC, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6_PERFCTR_BR_EXEC, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV6_PERFCTR_BR_MISPREDICT, - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV6_PERFCTR_IBUF_STALL, - [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV6_PERFCTR_LSU_FULL_STALL, -}; - -static const unsigned armv6_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - /* - * The performance counters don't differentiate between read and write - * accesses/misses so this isn't strictly correct, but it's the best we - * can do. Writes and reads get combined. - */ - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV6_PERFCTR_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_DCACHE_MISS, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV6_PERFCTR_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6_PERFCTR_DCACHE_MISS, - - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_ICACHE_MISS, - - /* - * The ARM performance counters can count micro DTLB misses, micro ITLB - * misses and main TLB misses. There isn't an event for TLB misses, so - * use the micro misses here and if users want the main TLB misses they - * can use a raw counter. - */ - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_DTLB_MISS, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6_PERFCTR_DTLB_MISS, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_ITLB_MISS, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6_PERFCTR_ITLB_MISS, -}; - -enum armv6mpcore_perf_types { - ARMV6MPCORE_PERFCTR_ICACHE_MISS = 0x0, - ARMV6MPCORE_PERFCTR_IBUF_STALL = 0x1, - ARMV6MPCORE_PERFCTR_DDEP_STALL = 0x2, - ARMV6MPCORE_PERFCTR_ITLB_MISS = 0x3, - ARMV6MPCORE_PERFCTR_DTLB_MISS = 0x4, - ARMV6MPCORE_PERFCTR_BR_EXEC = 0x5, - ARMV6MPCORE_PERFCTR_BR_NOTPREDICT = 0x6, - ARMV6MPCORE_PERFCTR_BR_MISPREDICT = 0x7, - ARMV6MPCORE_PERFCTR_INSTR_EXEC = 0x8, - ARMV6MPCORE_PERFCTR_DCACHE_RDACCESS = 0xA, - ARMV6MPCORE_PERFCTR_DCACHE_RDMISS = 0xB, - ARMV6MPCORE_PERFCTR_DCACHE_WRACCESS = 0xC, - ARMV6MPCORE_PERFCTR_DCACHE_WRMISS = 0xD, - ARMV6MPCORE_PERFCTR_DCACHE_EVICTION = 0xE, - ARMV6MPCORE_PERFCTR_SW_PC_CHANGE = 0xF, - ARMV6MPCORE_PERFCTR_MAIN_TLB_MISS = 0x10, - ARMV6MPCORE_PERFCTR_EXPL_MEM_ACCESS = 0x11, - ARMV6MPCORE_PERFCTR_LSU_FULL_STALL = 0x12, - ARMV6MPCORE_PERFCTR_WBUF_DRAINED = 0x13, - ARMV6MPCORE_PERFCTR_CPU_CYCLES = 0xFF, -}; - -/* - * The hardware events that we support. We do support cache operations but - * we have harvard caches and no way to combine instruction and data - * accesses/misses in hardware. - */ -static const unsigned armv6mpcore_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV6MPCORE_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV6MPCORE_PERFCTR_INSTR_EXEC, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6MPCORE_PERFCTR_BR_EXEC, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV6MPCORE_PERFCTR_BR_MISPREDICT, - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV6MPCORE_PERFCTR_IBUF_STALL, - [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV6MPCORE_PERFCTR_LSU_FULL_STALL, -}; - -static const unsigned armv6mpcore_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV6MPCORE_PERFCTR_DCACHE_RDACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_DCACHE_RDMISS, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV6MPCORE_PERFCTR_DCACHE_WRACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_DCACHE_WRMISS, - - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ICACHE_MISS, - - /* - * The ARM performance counters can count micro DTLB misses, micro ITLB - * misses and main TLB misses. There isn't an event for TLB misses, so - * use the micro misses here and if users want the main TLB misses they - * can use a raw counter. - */ - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_DTLB_MISS, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_DTLB_MISS, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ITLB_MISS, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ITLB_MISS, -}; - -static inline unsigned long -armv6_pmcr_read(void) -{ - u32 val; - asm volatile("mrc p15, 0, %0, c15, c12, 0" : "=r"(val)); - return val; -} - -static inline void -armv6_pmcr_write(unsigned long val) -{ - asm volatile("mcr p15, 0, %0, c15, c12, 0" : : "r"(val)); -} - -#define ARMV6_PMCR_ENABLE (1 << 0) -#define ARMV6_PMCR_CTR01_RESET (1 << 1) -#define ARMV6_PMCR_CCOUNT_RESET (1 << 2) -#define ARMV6_PMCR_CCOUNT_DIV (1 << 3) -#define ARMV6_PMCR_COUNT0_IEN (1 << 4) -#define ARMV6_PMCR_COUNT1_IEN (1 << 5) -#define ARMV6_PMCR_CCOUNT_IEN (1 << 6) -#define ARMV6_PMCR_COUNT0_OVERFLOW (1 << 8) -#define ARMV6_PMCR_COUNT1_OVERFLOW (1 << 9) -#define ARMV6_PMCR_CCOUNT_OVERFLOW (1 << 10) -#define ARMV6_PMCR_EVT_COUNT0_SHIFT 20 -#define ARMV6_PMCR_EVT_COUNT0_MASK (0xFF << ARMV6_PMCR_EVT_COUNT0_SHIFT) -#define ARMV6_PMCR_EVT_COUNT1_SHIFT 12 -#define ARMV6_PMCR_EVT_COUNT1_MASK (0xFF << ARMV6_PMCR_EVT_COUNT1_SHIFT) - -#define ARMV6_PMCR_OVERFLOWED_MASK \ - (ARMV6_PMCR_COUNT0_OVERFLOW | ARMV6_PMCR_COUNT1_OVERFLOW | \ - ARMV6_PMCR_CCOUNT_OVERFLOW) - -static inline int -armv6_pmcr_has_overflowed(unsigned long pmcr) -{ - return pmcr & ARMV6_PMCR_OVERFLOWED_MASK; -} - -static inline int -armv6_pmcr_counter_has_overflowed(unsigned long pmcr, - enum armv6_counters counter) -{ - int ret = 0; - - if (ARMV6_CYCLE_COUNTER == counter) - ret = pmcr & ARMV6_PMCR_CCOUNT_OVERFLOW; - else if (ARMV6_COUNTER0 == counter) - ret = pmcr & ARMV6_PMCR_COUNT0_OVERFLOW; - else if (ARMV6_COUNTER1 == counter) - ret = pmcr & ARMV6_PMCR_COUNT1_OVERFLOW; - else - WARN_ONCE(1, "invalid counter number (%d)\n", counter); - - return ret; -} - -static inline u64 armv6pmu_read_counter(struct perf_event *event) -{ - struct hw_perf_event *hwc = &event->hw; - int counter = hwc->idx; - unsigned long value = 0; - - if (ARMV6_CYCLE_COUNTER == counter) - asm volatile("mrc p15, 0, %0, c15, c12, 1" : "=r"(value)); - else if (ARMV6_COUNTER0 == counter) - asm volatile("mrc p15, 0, %0, c15, c12, 2" : "=r"(value)); - else if (ARMV6_COUNTER1 == counter) - asm volatile("mrc p15, 0, %0, c15, c12, 3" : "=r"(value)); - else - WARN_ONCE(1, "invalid counter number (%d)\n", counter); - - return value; -} - -static inline void armv6pmu_write_counter(struct perf_event *event, u64 value) -{ - struct hw_perf_event *hwc = &event->hw; - int counter = hwc->idx; - - if (ARMV6_CYCLE_COUNTER == counter) - asm volatile("mcr p15, 0, %0, c15, c12, 1" : : "r"(value)); - else if (ARMV6_COUNTER0 == counter) - asm volatile("mcr p15, 0, %0, c15, c12, 2" : : "r"(value)); - else if (ARMV6_COUNTER1 == counter) - asm volatile("mcr p15, 0, %0, c15, c12, 3" : : "r"(value)); - else - WARN_ONCE(1, "invalid counter number (%d)\n", counter); -} - -static void armv6pmu_enable_event(struct perf_event *event) -{ - unsigned long val, mask, evt, flags; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - if (ARMV6_CYCLE_COUNTER == idx) { - mask = 0; - evt = ARMV6_PMCR_CCOUNT_IEN; - } else if (ARMV6_COUNTER0 == idx) { - mask = ARMV6_PMCR_EVT_COUNT0_MASK; - evt = (hwc->config_base << ARMV6_PMCR_EVT_COUNT0_SHIFT) | - ARMV6_PMCR_COUNT0_IEN; - } else if (ARMV6_COUNTER1 == idx) { - mask = ARMV6_PMCR_EVT_COUNT1_MASK; - evt = (hwc->config_base << ARMV6_PMCR_EVT_COUNT1_SHIFT) | - ARMV6_PMCR_COUNT1_IEN; - } else { - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - /* - * Mask out the current event and set the counter to count the event - * that we're interested in. - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = armv6_pmcr_read(); - val &= ~mask; - val |= evt; - armv6_pmcr_write(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static irqreturn_t -armv6pmu_handle_irq(struct arm_pmu *cpu_pmu) -{ - unsigned long pmcr = armv6_pmcr_read(); - struct perf_sample_data data; - struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); - struct pt_regs *regs; - int idx; - - if (!armv6_pmcr_has_overflowed(pmcr)) - return IRQ_NONE; - - regs = get_irq_regs(); - - /* - * The interrupts are cleared by writing the overflow flags back to - * the control register. All of the other bits don't have any effect - * if they are rewritten, so write the whole value back. - */ - armv6_pmcr_write(pmcr); - - for (idx = 0; idx < cpu_pmu->num_events; ++idx) { - struct perf_event *event = cpuc->events[idx]; - struct hw_perf_event *hwc; - - /* Ignore if we don't have an event. */ - if (!event) - continue; - - /* - * We have a single interrupt for all counters. Check that - * each counter has overflowed before we process it. - */ - if (!armv6_pmcr_counter_has_overflowed(pmcr, idx)) - continue; - - hwc = &event->hw; - armpmu_event_update(event); - perf_sample_data_init(&data, 0, hwc->last_period); - if (!armpmu_event_set_period(event)) - continue; - - if (perf_event_overflow(event, &data, regs)) - cpu_pmu->disable(event); - } - - /* - * Handle the pending perf events. - * - * Note: this call *must* be run with interrupts disabled. For - * platforms that can have the PMU interrupts raised as an NMI, this - * will not work. - */ - irq_work_run(); - - return IRQ_HANDLED; -} - -static void armv6pmu_start(struct arm_pmu *cpu_pmu) -{ - unsigned long flags, val; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = armv6_pmcr_read(); - val |= ARMV6_PMCR_ENABLE; - armv6_pmcr_write(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void armv6pmu_stop(struct arm_pmu *cpu_pmu) -{ - unsigned long flags, val; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = armv6_pmcr_read(); - val &= ~ARMV6_PMCR_ENABLE; - armv6_pmcr_write(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static int -armv6pmu_get_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - struct hw_perf_event *hwc = &event->hw; - /* Always place a cycle counter into the cycle counter. */ - if (ARMV6_PERFCTR_CPU_CYCLES == hwc->config_base) { - if (test_and_set_bit(ARMV6_CYCLE_COUNTER, cpuc->used_mask)) - return -EAGAIN; - - return ARMV6_CYCLE_COUNTER; - } else { - /* - * For anything other than a cycle counter, try and use - * counter0 and counter1. - */ - if (!test_and_set_bit(ARMV6_COUNTER1, cpuc->used_mask)) - return ARMV6_COUNTER1; - - if (!test_and_set_bit(ARMV6_COUNTER0, cpuc->used_mask)) - return ARMV6_COUNTER0; - - /* The counters are all in use. */ - return -EAGAIN; - } -} - -static void armv6pmu_clear_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - clear_bit(event->hw.idx, cpuc->used_mask); -} - -static void armv6pmu_disable_event(struct perf_event *event) -{ - unsigned long val, mask, evt, flags; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - if (ARMV6_CYCLE_COUNTER == idx) { - mask = ARMV6_PMCR_CCOUNT_IEN; - evt = 0; - } else if (ARMV6_COUNTER0 == idx) { - mask = ARMV6_PMCR_COUNT0_IEN | ARMV6_PMCR_EVT_COUNT0_MASK; - evt = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT0_SHIFT; - } else if (ARMV6_COUNTER1 == idx) { - mask = ARMV6_PMCR_COUNT1_IEN | ARMV6_PMCR_EVT_COUNT1_MASK; - evt = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT1_SHIFT; - } else { - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - /* - * Mask out the current event and set the counter to count the number - * of ETM bus signal assertion cycles. The external reporting should - * be disabled and so this should never increment. - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = armv6_pmcr_read(); - val &= ~mask; - val |= evt; - armv6_pmcr_write(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void armv6mpcore_pmu_disable_event(struct perf_event *event) -{ - unsigned long val, mask, flags, evt = 0; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - if (ARMV6_CYCLE_COUNTER == idx) { - mask = ARMV6_PMCR_CCOUNT_IEN; - } else if (ARMV6_COUNTER0 == idx) { - mask = ARMV6_PMCR_COUNT0_IEN; - } else if (ARMV6_COUNTER1 == idx) { - mask = ARMV6_PMCR_COUNT1_IEN; - } else { - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - /* - * Unlike UP ARMv6, we don't have a way of stopping the counters. We - * simply disable the interrupt reporting. - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = armv6_pmcr_read(); - val &= ~mask; - val |= evt; - armv6_pmcr_write(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static int armv6_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv6_perf_map, - &armv6_perf_cache_map, 0xFF); -} - -static void armv6pmu_init(struct arm_pmu *cpu_pmu) -{ - cpu_pmu->handle_irq = armv6pmu_handle_irq; - cpu_pmu->enable = armv6pmu_enable_event; - cpu_pmu->disable = armv6pmu_disable_event; - cpu_pmu->read_counter = armv6pmu_read_counter; - cpu_pmu->write_counter = armv6pmu_write_counter; - cpu_pmu->get_event_idx = armv6pmu_get_event_idx; - cpu_pmu->clear_event_idx = armv6pmu_clear_event_idx; - cpu_pmu->start = armv6pmu_start; - cpu_pmu->stop = armv6pmu_stop; - cpu_pmu->map_event = armv6_map_event; - cpu_pmu->num_events = 3; -} - -static int armv6_1136_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv6pmu_init(cpu_pmu); - cpu_pmu->name = "armv6_1136"; - return 0; -} - -static int armv6_1156_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv6pmu_init(cpu_pmu); - cpu_pmu->name = "armv6_1156"; - return 0; -} - -static int armv6_1176_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv6pmu_init(cpu_pmu); - cpu_pmu->name = "armv6_1176"; - return 0; -} - -/* - * ARMv6mpcore is almost identical to single core ARMv6 with the exception - * that some of the events have different enumerations and that there is no - * *hack* to stop the programmable counters. To stop the counters we simply - * disable the interrupt reporting and update the event. When unthrottling we - * reset the period and enable the interrupt reporting. - */ - -static int armv6mpcore_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv6mpcore_perf_map, - &armv6mpcore_perf_cache_map, 0xFF); -} - -static int armv6mpcore_pmu_init(struct arm_pmu *cpu_pmu) -{ - cpu_pmu->name = "armv6_11mpcore"; - cpu_pmu->handle_irq = armv6pmu_handle_irq; - cpu_pmu->enable = armv6pmu_enable_event; - cpu_pmu->disable = armv6mpcore_pmu_disable_event; - cpu_pmu->read_counter = armv6pmu_read_counter; - cpu_pmu->write_counter = armv6pmu_write_counter; - cpu_pmu->get_event_idx = armv6pmu_get_event_idx; - cpu_pmu->clear_event_idx = armv6pmu_clear_event_idx; - cpu_pmu->start = armv6pmu_start; - cpu_pmu->stop = armv6pmu_stop; - cpu_pmu->map_event = armv6mpcore_map_event; - cpu_pmu->num_events = 3; - - return 0; -} - -static const struct of_device_id armv6_pmu_of_device_ids[] = { - {.compatible = "arm,arm11mpcore-pmu", .data = armv6mpcore_pmu_init}, - {.compatible = "arm,arm1176-pmu", .data = armv6_1176_pmu_init}, - {.compatible = "arm,arm1136-pmu", .data = armv6_1136_pmu_init}, - { /* sentinel value */ } -}; - -static const struct pmu_probe_info armv6_pmu_probe_table[] = { - ARM_PMU_PROBE(ARM_CPU_PART_ARM1136, armv6_1136_pmu_init), - ARM_PMU_PROBE(ARM_CPU_PART_ARM1156, armv6_1156_pmu_init), - ARM_PMU_PROBE(ARM_CPU_PART_ARM1176, armv6_1176_pmu_init), - ARM_PMU_PROBE(ARM_CPU_PART_ARM11MPCORE, armv6mpcore_pmu_init), - { /* sentinel value */ } -}; - -static int armv6_pmu_device_probe(struct platform_device *pdev) -{ - return arm_pmu_device_probe(pdev, armv6_pmu_of_device_ids, - armv6_pmu_probe_table); -} - -static struct platform_driver armv6_pmu_driver = { - .driver = { - .name = "armv6-pmu", - .of_match_table = armv6_pmu_of_device_ids, - }, - .probe = armv6_pmu_device_probe, -}; - -builtin_platform_driver(armv6_pmu_driver); -#endif /* CONFIG_CPU_V6 || CONFIG_CPU_V6K */ diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c deleted file mode 100644 index eb2190477da1..000000000000 --- a/arch/arm/kernel/perf_event_v7.c +++ /dev/null @@ -1,2047 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code. - * - * ARMv7 support: Jean Pihet <jpihet@mvista.com> - * 2010 (c) MontaVista Software, LLC. - * - * Copied from ARMv6 code, with the low level code inspired - * by the ARMv7 Oprofile code. - * - * Cortex-A8 has up to 4 configurable performance counters and - * a single cycle counter. - * Cortex-A9 has up to 31 configurable performance counters and - * a single cycle counter. - * - * All counters can be enabled/disabled and IRQ masked separately. The cycle - * counter and all 4 performance counters together can be reset separately. - */ - -#ifdef CONFIG_CPU_V7 - -#include <asm/cp15.h> -#include <asm/cputype.h> -#include <asm/irq_regs.h> -#include <asm/vfp.h> -#include "../vfp/vfpinstr.h" - -#include <linux/of.h> -#include <linux/perf/arm_pmu.h> -#include <linux/platform_device.h> - -/* - * Common ARMv7 event types - * - * Note: An implementation may not be able to count all of these events - * but the encodings are considered to be `reserved' in the case that - * they are not available. - */ -#define ARMV7_PERFCTR_PMNC_SW_INCR 0x00 -#define ARMV7_PERFCTR_L1_ICACHE_REFILL 0x01 -#define ARMV7_PERFCTR_ITLB_REFILL 0x02 -#define ARMV7_PERFCTR_L1_DCACHE_REFILL 0x03 -#define ARMV7_PERFCTR_L1_DCACHE_ACCESS 0x04 -#define ARMV7_PERFCTR_DTLB_REFILL 0x05 -#define ARMV7_PERFCTR_MEM_READ 0x06 -#define ARMV7_PERFCTR_MEM_WRITE 0x07 -#define ARMV7_PERFCTR_INSTR_EXECUTED 0x08 -#define ARMV7_PERFCTR_EXC_TAKEN 0x09 -#define ARMV7_PERFCTR_EXC_EXECUTED 0x0A -#define ARMV7_PERFCTR_CID_WRITE 0x0B - -/* - * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS. - * It counts: - * - all (taken) branch instructions, - * - instructions that explicitly write the PC, - * - exception generating instructions. - */ -#define ARMV7_PERFCTR_PC_WRITE 0x0C -#define ARMV7_PERFCTR_PC_IMM_BRANCH 0x0D -#define ARMV7_PERFCTR_PC_PROC_RETURN 0x0E -#define ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS 0x0F -#define ARMV7_PERFCTR_PC_BRANCH_MIS_PRED 0x10 -#define ARMV7_PERFCTR_CLOCK_CYCLES 0x11 -#define ARMV7_PERFCTR_PC_BRANCH_PRED 0x12 - -/* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */ -#define ARMV7_PERFCTR_MEM_ACCESS 0x13 -#define ARMV7_PERFCTR_L1_ICACHE_ACCESS 0x14 -#define ARMV7_PERFCTR_L1_DCACHE_WB 0x15 -#define ARMV7_PERFCTR_L2_CACHE_ACCESS 0x16 -#define ARMV7_PERFCTR_L2_CACHE_REFILL 0x17 -#define ARMV7_PERFCTR_L2_CACHE_WB 0x18 -#define ARMV7_PERFCTR_BUS_ACCESS 0x19 -#define ARMV7_PERFCTR_MEM_ERROR 0x1A -#define ARMV7_PERFCTR_INSTR_SPEC 0x1B -#define ARMV7_PERFCTR_TTBR_WRITE 0x1C -#define ARMV7_PERFCTR_BUS_CYCLES 0x1D - -#define ARMV7_PERFCTR_CPU_CYCLES 0xFF - -/* ARMv7 Cortex-A8 specific event types */ -#define ARMV7_A8_PERFCTR_L2_CACHE_ACCESS 0x43 -#define ARMV7_A8_PERFCTR_L2_CACHE_REFILL 0x44 -#define ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS 0x50 -#define ARMV7_A8_PERFCTR_STALL_ISIDE 0x56 - -/* ARMv7 Cortex-A9 specific event types */ -#define ARMV7_A9_PERFCTR_INSTR_CORE_RENAME 0x68 -#define ARMV7_A9_PERFCTR_STALL_ICACHE 0x60 -#define ARMV7_A9_PERFCTR_STALL_DISPATCH 0x66 - -/* ARMv7 Cortex-A5 specific event types */ -#define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL 0xc2 -#define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP 0xc3 - -/* ARMv7 Cortex-A15 specific event types */ -#define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ 0x40 -#define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE 0x41 -#define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ 0x42 -#define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE 0x43 - -#define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ 0x4C -#define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE 0x4D - -#define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ 0x50 -#define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE 0x51 -#define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ 0x52 -#define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE 0x53 - -#define ARMV7_A15_PERFCTR_PC_WRITE_SPEC 0x76 - -/* ARMv7 Cortex-A12 specific event types */ -#define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ 0x40 -#define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE 0x41 - -#define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ 0x50 -#define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE 0x51 - -#define ARMV7_A12_PERFCTR_PC_WRITE_SPEC 0x76 - -#define ARMV7_A12_PERFCTR_PF_TLB_REFILL 0xe7 - -/* ARMv7 Krait specific event types */ -#define KRAIT_PMRESR0_GROUP0 0xcc -#define KRAIT_PMRESR1_GROUP0 0xd0 -#define KRAIT_PMRESR2_GROUP0 0xd4 -#define KRAIT_VPMRESR0_GROUP0 0xd8 - -#define KRAIT_PERFCTR_L1_ICACHE_ACCESS 0x10011 -#define KRAIT_PERFCTR_L1_ICACHE_MISS 0x10010 - -#define KRAIT_PERFCTR_L1_ITLB_ACCESS 0x12222 -#define KRAIT_PERFCTR_L1_DTLB_ACCESS 0x12210 - -/* ARMv7 Scorpion specific event types */ -#define SCORPION_LPM0_GROUP0 0x4c -#define SCORPION_LPM1_GROUP0 0x50 -#define SCORPION_LPM2_GROUP0 0x54 -#define SCORPION_L2LPM_GROUP0 0x58 -#define SCORPION_VLPM_GROUP0 0x5c - -#define SCORPION_ICACHE_ACCESS 0x10053 -#define SCORPION_ICACHE_MISS 0x10052 - -#define SCORPION_DTLB_ACCESS 0x12013 -#define SCORPION_DTLB_MISS 0x12012 - -#define SCORPION_ITLB_MISS 0x12021 - -/* - * Cortex-A8 HW events mapping - * - * The hardware events that we support. We do support cache operations but - * we have harvard caches and no way to combine instruction and data - * accesses/misses in hardware. - */ -static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE, -}; - -static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - /* - * The performance counters don't differentiate between read and write - * accesses/misses so this isn't strictly correct, but it's the best we - * can do. Writes and reads get combined. - */ - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, - - [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS, - [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL, - [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS, - [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Cortex-A9 HW events mapping - */ -static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME, - [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE, - [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV7_A9_PERFCTR_STALL_DISPATCH, -}; - -static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - /* - * The performance counters don't differentiate between read and write - * accesses/misses so this isn't strictly correct, but it's the best we - * can do. Writes and reads get combined. - */ - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Cortex-A5 HW events mapping - */ -static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL, - [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP, - - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, - /* - * The prefetch counters don't differentiate between the I side and the - * D side. - */ - [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL, - [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Cortex-A15 HW events mapping - */ -static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A15_PERFCTR_PC_WRITE_SPEC, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, -}; - -static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE, - - /* - * Not all performance counters differentiate between read and write - * accesses/misses so we're not always strictly correct, but it's the - * best we can do. Writes and reads get combined in these cases. - */ - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, - - [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ, - [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ, - [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE, - [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Cortex-A7 HW events mapping - */ -static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, -}; - -static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - /* - * The performance counters don't differentiate between read and write - * accesses/misses so this isn't strictly correct, but it's the best we - * can do. Writes and reads get combined. - */ - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, - - [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS, - [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, - [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS, - [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Cortex-A12 HW events mapping - */ -static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A12_PERFCTR_PC_WRITE_SPEC, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, -}; - -static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - - /* - * Not all performance counters differentiate between read and write - * accesses/misses so we're not always strictly correct, but it's the - * best we can do. Writes and reads get combined in these cases. - */ - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, - - [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ, - [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, - [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE, - [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A12_PERFCTR_PF_TLB_REFILL, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Krait HW events mapping - */ -static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, -}; - -static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, -}; - -static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - /* - * The performance counters don't differentiate between read and write - * accesses/misses so this isn't strictly correct, but it's the best we - * can do. Writes and reads get combined. - */ - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = KRAIT_PERFCTR_L1_ICACHE_MISS, - - [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS, - [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS, - - [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS, - [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Scorpion HW events mapping - */ -static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, -}; - -static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - /* - * The performance counters don't differentiate between read and write - * accesses/misses so this isn't strictly correct, but it's the best we - * can do. Writes and reads get combined. - */ - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS, - /* - * Only ITLB misses and DTLB refills are supported. If users want the - * DTLB refills misses a raw counter must be used. - */ - [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS, - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS, - [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS, - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS, - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -PMU_FORMAT_ATTR(event, "config:0-7"); - -static struct attribute *armv7_pmu_format_attrs[] = { - &format_attr_event.attr, - NULL, -}; - -static struct attribute_group armv7_pmu_format_attr_group = { - .name = "format", - .attrs = armv7_pmu_format_attrs, -}; - -#define ARMV7_EVENT_ATTR_RESOLVE(m) #m -#define ARMV7_EVENT_ATTR(name, config) \ - PMU_EVENT_ATTR_STRING(name, armv7_event_attr_##name, \ - "event=" ARMV7_EVENT_ATTR_RESOLVE(config)) - -ARMV7_EVENT_ATTR(sw_incr, ARMV7_PERFCTR_PMNC_SW_INCR); -ARMV7_EVENT_ATTR(l1i_cache_refill, ARMV7_PERFCTR_L1_ICACHE_REFILL); -ARMV7_EVENT_ATTR(l1i_tlb_refill, ARMV7_PERFCTR_ITLB_REFILL); -ARMV7_EVENT_ATTR(l1d_cache_refill, ARMV7_PERFCTR_L1_DCACHE_REFILL); -ARMV7_EVENT_ATTR(l1d_cache, ARMV7_PERFCTR_L1_DCACHE_ACCESS); -ARMV7_EVENT_ATTR(l1d_tlb_refill, ARMV7_PERFCTR_DTLB_REFILL); -ARMV7_EVENT_ATTR(ld_retired, ARMV7_PERFCTR_MEM_READ); -ARMV7_EVENT_ATTR(st_retired, ARMV7_PERFCTR_MEM_WRITE); -ARMV7_EVENT_ATTR(inst_retired, ARMV7_PERFCTR_INSTR_EXECUTED); -ARMV7_EVENT_ATTR(exc_taken, ARMV7_PERFCTR_EXC_TAKEN); -ARMV7_EVENT_ATTR(exc_return, ARMV7_PERFCTR_EXC_EXECUTED); -ARMV7_EVENT_ATTR(cid_write_retired, ARMV7_PERFCTR_CID_WRITE); -ARMV7_EVENT_ATTR(pc_write_retired, ARMV7_PERFCTR_PC_WRITE); -ARMV7_EVENT_ATTR(br_immed_retired, ARMV7_PERFCTR_PC_IMM_BRANCH); -ARMV7_EVENT_ATTR(br_return_retired, ARMV7_PERFCTR_PC_PROC_RETURN); -ARMV7_EVENT_ATTR(unaligned_ldst_retired, ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS); -ARMV7_EVENT_ATTR(br_mis_pred, ARMV7_PERFCTR_PC_BRANCH_MIS_PRED); -ARMV7_EVENT_ATTR(cpu_cycles, ARMV7_PERFCTR_CLOCK_CYCLES); -ARMV7_EVENT_ATTR(br_pred, ARMV7_PERFCTR_PC_BRANCH_PRED); - -static struct attribute *armv7_pmuv1_event_attrs[] = { - &armv7_event_attr_sw_incr.attr.attr, - &armv7_event_attr_l1i_cache_refill.attr.attr, - &armv7_event_attr_l1i_tlb_refill.attr.attr, - &armv7_event_attr_l1d_cache_refill.attr.attr, - &armv7_event_attr_l1d_cache.attr.attr, - &armv7_event_attr_l1d_tlb_refill.attr.attr, - &armv7_event_attr_ld_retired.attr.attr, - &armv7_event_attr_st_retired.attr.attr, - &armv7_event_attr_inst_retired.attr.attr, - &armv7_event_attr_exc_taken.attr.attr, - &armv7_event_attr_exc_return.attr.attr, - &armv7_event_attr_cid_write_retired.attr.attr, - &armv7_event_attr_pc_write_retired.attr.attr, - &armv7_event_attr_br_immed_retired.attr.attr, - &armv7_event_attr_br_return_retired.attr.attr, - &armv7_event_attr_unaligned_ldst_retired.attr.attr, - &armv7_event_attr_br_mis_pred.attr.attr, - &armv7_event_attr_cpu_cycles.attr.attr, - &armv7_event_attr_br_pred.attr.attr, - NULL, -}; - -static struct attribute_group armv7_pmuv1_events_attr_group = { - .name = "events", - .attrs = armv7_pmuv1_event_attrs, -}; - -ARMV7_EVENT_ATTR(mem_access, ARMV7_PERFCTR_MEM_ACCESS); -ARMV7_EVENT_ATTR(l1i_cache, ARMV7_PERFCTR_L1_ICACHE_ACCESS); -ARMV7_EVENT_ATTR(l1d_cache_wb, ARMV7_PERFCTR_L1_DCACHE_WB); -ARMV7_EVENT_ATTR(l2d_cache, ARMV7_PERFCTR_L2_CACHE_ACCESS); -ARMV7_EVENT_ATTR(l2d_cache_refill, ARMV7_PERFCTR_L2_CACHE_REFILL); -ARMV7_EVENT_ATTR(l2d_cache_wb, ARMV7_PERFCTR_L2_CACHE_WB); -ARMV7_EVENT_ATTR(bus_access, ARMV7_PERFCTR_BUS_ACCESS); -ARMV7_EVENT_ATTR(memory_error, ARMV7_PERFCTR_MEM_ERROR); -ARMV7_EVENT_ATTR(inst_spec, ARMV7_PERFCTR_INSTR_SPEC); -ARMV7_EVENT_ATTR(ttbr_write_retired, ARMV7_PERFCTR_TTBR_WRITE); -ARMV7_EVENT_ATTR(bus_cycles, ARMV7_PERFCTR_BUS_CYCLES); - -static struct attribute *armv7_pmuv2_event_attrs[] = { - &armv7_event_attr_sw_incr.attr.attr, - &armv7_event_attr_l1i_cache_refill.attr.attr, - &armv7_event_attr_l1i_tlb_refill.attr.attr, - &armv7_event_attr_l1d_cache_refill.attr.attr, - &armv7_event_attr_l1d_cache.attr.attr, - &armv7_event_attr_l1d_tlb_refill.attr.attr, - &armv7_event_attr_ld_retired.attr.attr, - &armv7_event_attr_st_retired.attr.attr, - &armv7_event_attr_inst_retired.attr.attr, - &armv7_event_attr_exc_taken.attr.attr, - &armv7_event_attr_exc_return.attr.attr, - &armv7_event_attr_cid_write_retired.attr.attr, - &armv7_event_attr_pc_write_retired.attr.attr, - &armv7_event_attr_br_immed_retired.attr.attr, - &armv7_event_attr_br_return_retired.attr.attr, - &armv7_event_attr_unaligned_ldst_retired.attr.attr, - &armv7_event_attr_br_mis_pred.attr.attr, - &armv7_event_attr_cpu_cycles.attr.attr, - &armv7_event_attr_br_pred.attr.attr, - &armv7_event_attr_mem_access.attr.attr, - &armv7_event_attr_l1i_cache.attr.attr, - &armv7_event_attr_l1d_cache_wb.attr.attr, - &armv7_event_attr_l2d_cache.attr.attr, - &armv7_event_attr_l2d_cache_refill.attr.attr, - &armv7_event_attr_l2d_cache_wb.attr.attr, - &armv7_event_attr_bus_access.attr.attr, - &armv7_event_attr_memory_error.attr.attr, - &armv7_event_attr_inst_spec.attr.attr, - &armv7_event_attr_ttbr_write_retired.attr.attr, - &armv7_event_attr_bus_cycles.attr.attr, - NULL, -}; - -static struct attribute_group armv7_pmuv2_events_attr_group = { - .name = "events", - .attrs = armv7_pmuv2_event_attrs, -}; - -/* - * Perf Events' indices - */ -#define ARMV7_IDX_CYCLE_COUNTER 0 -#define ARMV7_IDX_COUNTER0 1 -#define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \ - (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1) - -#define ARMV7_MAX_COUNTERS 32 -#define ARMV7_COUNTER_MASK (ARMV7_MAX_COUNTERS - 1) - -/* - * ARMv7 low level PMNC access - */ - -/* - * Perf Event to low level counters mapping - */ -#define ARMV7_IDX_TO_COUNTER(x) \ - (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK) - -/* - * Per-CPU PMNC: config reg - */ -#define ARMV7_PMNC_E (1 << 0) /* Enable all counters */ -#define ARMV7_PMNC_P (1 << 1) /* Reset all counters */ -#define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */ -#define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */ -#define ARMV7_PMNC_X (1 << 4) /* Export to ETM */ -#define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/ -#define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */ -#define ARMV7_PMNC_N_MASK 0x1f -#define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */ - -/* - * FLAG: counters overflow flag status reg - */ -#define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */ -#define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK - -/* - * PMXEVTYPER: Event selection reg - */ -#define ARMV7_EVTYPE_MASK 0xc80000ff /* Mask for writable bits */ -#define ARMV7_EVTYPE_EVENT 0xff /* Mask for EVENT bits */ - -/* - * Event filters for PMUv2 - */ -#define ARMV7_EXCLUDE_PL1 BIT(31) -#define ARMV7_EXCLUDE_USER BIT(30) -#define ARMV7_INCLUDE_HYP BIT(27) - -/* - * Secure debug enable reg - */ -#define ARMV7_SDER_SUNIDEN BIT(1) /* Permit non-invasive debug */ - -static inline u32 armv7_pmnc_read(void) -{ - u32 val; - asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val)); - return val; -} - -static inline void armv7_pmnc_write(u32 val) -{ - val &= ARMV7_PMNC_MASK; - isb(); - asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val)); -} - -static inline int armv7_pmnc_has_overflowed(u32 pmnc) -{ - return pmnc & ARMV7_OVERFLOWED_MASK; -} - -static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx) -{ - return idx >= ARMV7_IDX_CYCLE_COUNTER && - idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); -} - -static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx) -{ - return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx)); -} - -static inline void armv7_pmnc_select_counter(int idx) -{ - u32 counter = ARMV7_IDX_TO_COUNTER(idx); - asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter)); - isb(); -} - -static inline u64 armv7pmu_read_counter(struct perf_event *event) -{ - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - u32 value = 0; - - if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { - pr_err("CPU%u reading wrong counter %d\n", - smp_processor_id(), idx); - } else if (idx == ARMV7_IDX_CYCLE_COUNTER) { - asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value)); - } else { - armv7_pmnc_select_counter(idx); - asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value)); - } - - return value; -} - -static inline void armv7pmu_write_counter(struct perf_event *event, u64 value) -{ - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - - if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { - pr_err("CPU%u writing wrong counter %d\n", - smp_processor_id(), idx); - } else if (idx == ARMV7_IDX_CYCLE_COUNTER) { - asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" ((u32)value)); - } else { - armv7_pmnc_select_counter(idx); - asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" ((u32)value)); - } -} - -static inline void armv7_pmnc_write_evtsel(int idx, u32 val) -{ - armv7_pmnc_select_counter(idx); - val &= ARMV7_EVTYPE_MASK; - asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val)); -} - -static inline void armv7_pmnc_enable_counter(int idx) -{ - u32 counter = ARMV7_IDX_TO_COUNTER(idx); - asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter))); -} - -static inline void armv7_pmnc_disable_counter(int idx) -{ - u32 counter = ARMV7_IDX_TO_COUNTER(idx); - asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter))); -} - -static inline void armv7_pmnc_enable_intens(int idx) -{ - u32 counter = ARMV7_IDX_TO_COUNTER(idx); - asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter))); -} - -static inline void armv7_pmnc_disable_intens(int idx) -{ - u32 counter = ARMV7_IDX_TO_COUNTER(idx); - asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter))); - isb(); - /* Clear the overflow flag in case an interrupt is pending. */ - asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter))); - isb(); -} - -static inline u32 armv7_pmnc_getreset_flags(void) -{ - u32 val; - - /* Read */ - asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); - - /* Write to clear flags */ - val &= ARMV7_FLAG_MASK; - asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val)); - - return val; -} - -#ifdef DEBUG -static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu) -{ - u32 val; - unsigned int cnt; - - pr_info("PMNC registers dump:\n"); - - asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); - pr_info("PMNC =0x%08x\n", val); - - asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val)); - pr_info("CNTENS=0x%08x\n", val); - - asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val)); - pr_info("INTENS=0x%08x\n", val); - - asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); - pr_info("FLAGS =0x%08x\n", val); - - asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val)); - pr_info("SELECT=0x%08x\n", val); - - asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); - pr_info("CCNT =0x%08x\n", val); - - for (cnt = ARMV7_IDX_COUNTER0; - cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) { - armv7_pmnc_select_counter(cnt); - asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); - pr_info("CNT[%d] count =0x%08x\n", - ARMV7_IDX_TO_COUNTER(cnt), val); - asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val)); - pr_info("CNT[%d] evtsel=0x%08x\n", - ARMV7_IDX_TO_COUNTER(cnt), val); - } -} -#endif - -static void armv7pmu_enable_event(struct perf_event *event) -{ - unsigned long flags; - struct hw_perf_event *hwc = &event->hw; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { - pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n", - smp_processor_id(), idx); - return; - } - - /* - * Enable counter and interrupt, and set the counter to count - * the event that we're interested in. - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* - * Disable counter - */ - armv7_pmnc_disable_counter(idx); - - /* - * Set event (if destined for PMNx counters) - * We only need to set the event for the cycle counter if we - * have the ability to perform event filtering. - */ - if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER) - armv7_pmnc_write_evtsel(idx, hwc->config_base); - - /* - * Enable interrupt for this counter - */ - armv7_pmnc_enable_intens(idx); - - /* - * Enable counter - */ - armv7_pmnc_enable_counter(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void armv7pmu_disable_event(struct perf_event *event) -{ - unsigned long flags; - struct hw_perf_event *hwc = &event->hw; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { - pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n", - smp_processor_id(), idx); - return; - } - - /* - * Disable counter and interrupt - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* - * Disable counter - */ - armv7_pmnc_disable_counter(idx); - - /* - * Disable interrupt for this counter - */ - armv7_pmnc_disable_intens(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static irqreturn_t armv7pmu_handle_irq(struct arm_pmu *cpu_pmu) -{ - u32 pmnc; - struct perf_sample_data data; - struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); - struct pt_regs *regs; - int idx; - - /* - * Get and reset the IRQ flags - */ - pmnc = armv7_pmnc_getreset_flags(); - - /* - * Did an overflow occur? - */ - if (!armv7_pmnc_has_overflowed(pmnc)) - return IRQ_NONE; - - /* - * Handle the counter(s) overflow(s) - */ - regs = get_irq_regs(); - - for (idx = 0; idx < cpu_pmu->num_events; ++idx) { - struct perf_event *event = cpuc->events[idx]; - struct hw_perf_event *hwc; - - /* Ignore if we don't have an event. */ - if (!event) - continue; - - /* - * We have a single interrupt for all counters. Check that - * each counter has overflowed before we process it. - */ - if (!armv7_pmnc_counter_has_overflowed(pmnc, idx)) - continue; - - hwc = &event->hw; - armpmu_event_update(event); - perf_sample_data_init(&data, 0, hwc->last_period); - if (!armpmu_event_set_period(event)) - continue; - - if (perf_event_overflow(event, &data, regs)) - cpu_pmu->disable(event); - } - - /* - * Handle the pending perf events. - * - * Note: this call *must* be run with interrupts disabled. For - * platforms that can have the PMU interrupts raised as an NMI, this - * will not work. - */ - irq_work_run(); - - return IRQ_HANDLED; -} - -static void armv7pmu_start(struct arm_pmu *cpu_pmu) -{ - unsigned long flags; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - /* Enable all counters */ - armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void armv7pmu_stop(struct arm_pmu *cpu_pmu) -{ - unsigned long flags; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - /* Disable all counters */ - armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - int idx; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT; - - /* Always place a cycle counter into the cycle counter. */ - if (evtype == ARMV7_PERFCTR_CPU_CYCLES) { - if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask)) - return -EAGAIN; - - return ARMV7_IDX_CYCLE_COUNTER; - } - - /* - * For anything other than a cycle counter, try and use - * the events counters - */ - for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) { - if (!test_and_set_bit(idx, cpuc->used_mask)) - return idx; - } - - /* The counters are all in use. */ - return -EAGAIN; -} - -static void armv7pmu_clear_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - clear_bit(event->hw.idx, cpuc->used_mask); -} - -/* - * Add an event filter to a given event. This will only work for PMUv2 PMUs. - */ -static int armv7pmu_set_event_filter(struct hw_perf_event *event, - struct perf_event_attr *attr) -{ - unsigned long config_base = 0; - - if (attr->exclude_idle) - return -EPERM; - if (attr->exclude_user) - config_base |= ARMV7_EXCLUDE_USER; - if (attr->exclude_kernel) - config_base |= ARMV7_EXCLUDE_PL1; - if (!attr->exclude_hv) - config_base |= ARMV7_INCLUDE_HYP; - - /* - * Install the filter into config_base as this is used to - * construct the event type. - */ - event->config_base = config_base; - - return 0; -} - -static void armv7pmu_reset(void *info) -{ - struct arm_pmu *cpu_pmu = (struct arm_pmu *)info; - u32 idx, nb_cnt = cpu_pmu->num_events, val; - - if (cpu_pmu->secure_access) { - asm volatile("mrc p15, 0, %0, c1, c1, 1" : "=r" (val)); - val |= ARMV7_SDER_SUNIDEN; - asm volatile("mcr p15, 0, %0, c1, c1, 1" : : "r" (val)); - } - - /* The counter and interrupt enable registers are unknown at reset. */ - for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { - armv7_pmnc_disable_counter(idx); - armv7_pmnc_disable_intens(idx); - } - - /* Initialize & Reset PMNC: C and P bits */ - armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C); -} - -static int armv7_a8_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv7_a8_perf_map, - &armv7_a8_perf_cache_map, 0xFF); -} - -static int armv7_a9_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv7_a9_perf_map, - &armv7_a9_perf_cache_map, 0xFF); -} - -static int armv7_a5_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv7_a5_perf_map, - &armv7_a5_perf_cache_map, 0xFF); -} - -static int armv7_a15_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv7_a15_perf_map, - &armv7_a15_perf_cache_map, 0xFF); -} - -static int armv7_a7_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv7_a7_perf_map, - &armv7_a7_perf_cache_map, 0xFF); -} - -static int armv7_a12_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv7_a12_perf_map, - &armv7_a12_perf_cache_map, 0xFF); -} - -static int krait_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &krait_perf_map, - &krait_perf_cache_map, 0xFFFFF); -} - -static int krait_map_event_no_branch(struct perf_event *event) -{ - return armpmu_map_event(event, &krait_perf_map_no_branch, - &krait_perf_cache_map, 0xFFFFF); -} - -static int scorpion_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &scorpion_perf_map, - &scorpion_perf_cache_map, 0xFFFFF); -} - -static void armv7pmu_init(struct arm_pmu *cpu_pmu) -{ - cpu_pmu->handle_irq = armv7pmu_handle_irq; - cpu_pmu->enable = armv7pmu_enable_event; - cpu_pmu->disable = armv7pmu_disable_event; - cpu_pmu->read_counter = armv7pmu_read_counter; - cpu_pmu->write_counter = armv7pmu_write_counter; - cpu_pmu->get_event_idx = armv7pmu_get_event_idx; - cpu_pmu->clear_event_idx = armv7pmu_clear_event_idx; - cpu_pmu->start = armv7pmu_start; - cpu_pmu->stop = armv7pmu_stop; - cpu_pmu->reset = armv7pmu_reset; -}; - -static void armv7_read_num_pmnc_events(void *info) -{ - int *nb_cnt = info; - - /* Read the nb of CNTx counters supported from PMNC */ - *nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK; - - /* Add the CPU cycles counter */ - *nb_cnt += 1; -} - -static int armv7_probe_num_events(struct arm_pmu *arm_pmu) -{ - return smp_call_function_any(&arm_pmu->supported_cpus, - armv7_read_num_pmnc_events, - &arm_pmu->num_events, 1); -} - -static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a8"; - cpu_pmu->map_event = armv7_a8_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv1_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return armv7_probe_num_events(cpu_pmu); -} - -static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a9"; - cpu_pmu->map_event = armv7_a9_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv1_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return armv7_probe_num_events(cpu_pmu); -} - -static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a5"; - cpu_pmu->map_event = armv7_a5_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv1_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return armv7_probe_num_events(cpu_pmu); -} - -static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a15"; - cpu_pmu->map_event = armv7_a15_map_event; - cpu_pmu->set_event_filter = armv7pmu_set_event_filter; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv2_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return armv7_probe_num_events(cpu_pmu); -} - -static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a7"; - cpu_pmu->map_event = armv7_a7_map_event; - cpu_pmu->set_event_filter = armv7pmu_set_event_filter; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv2_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return armv7_probe_num_events(cpu_pmu); -} - -static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a12"; - cpu_pmu->map_event = armv7_a12_map_event; - cpu_pmu->set_event_filter = armv7pmu_set_event_filter; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv2_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return armv7_probe_num_events(cpu_pmu); -} - -static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu) -{ - int ret = armv7_a12_pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a17"; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv2_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return ret; -} - -/* - * Krait Performance Monitor Region Event Selection Register (PMRESRn) - * - * 31 30 24 16 8 0 - * +--------------------------------+ - * PMRESR0 | EN | CC | CC | CC | CC | N = 1, R = 0 - * +--------------------------------+ - * PMRESR1 | EN | CC | CC | CC | CC | N = 1, R = 1 - * +--------------------------------+ - * PMRESR2 | EN | CC | CC | CC | CC | N = 1, R = 2 - * +--------------------------------+ - * VPMRESR0 | EN | CC | CC | CC | CC | N = 2, R = ? - * +--------------------------------+ - * EN | G=3 | G=2 | G=1 | G=0 - * - * Event Encoding: - * - * hwc->config_base = 0xNRCCG - * - * N = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR) - * R = region register - * CC = class of events the group G is choosing from - * G = group or particular event - * - * Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2 - * - * A region (R) corresponds to a piece of the CPU (execution unit, instruction - * unit, etc.) while the event code (CC) corresponds to a particular class of - * events (interrupts for example). An event code is broken down into - * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for - * example). - */ - -#define KRAIT_EVENT (1 << 16) -#define VENUM_EVENT (2 << 16) -#define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT) -#define PMRESRn_EN BIT(31) - -#define EVENT_REGION(event) (((event) >> 12) & 0xf) /* R */ -#define EVENT_GROUP(event) ((event) & 0xf) /* G */ -#define EVENT_CODE(event) (((event) >> 4) & 0xff) /* CC */ -#define EVENT_VENUM(event) (!!(event & VENUM_EVENT)) /* N=2 */ -#define EVENT_CPU(event) (!!(event & KRAIT_EVENT)) /* N=1 */ - -static u32 krait_read_pmresrn(int n) -{ - u32 val; - - switch (n) { - case 0: - asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val)); - break; - case 1: - asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val)); - break; - case 2: - asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val)); - break; - default: - BUG(); /* Should be validated in krait_pmu_get_event_idx() */ - } - - return val; -} - -static void krait_write_pmresrn(int n, u32 val) -{ - switch (n) { - case 0: - asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val)); - break; - case 1: - asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val)); - break; - case 2: - asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val)); - break; - default: - BUG(); /* Should be validated in krait_pmu_get_event_idx() */ - } -} - -static u32 venum_read_pmresr(void) -{ - u32 val; - asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val)); - return val; -} - -static void venum_write_pmresr(u32 val) -{ - asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val)); -} - -static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val) -{ - u32 venum_new_val; - u32 fp_new_val; - - BUG_ON(preemptible()); - /* CPACR Enable CP10 and CP11 access */ - *venum_orig_val = get_copro_access(); - venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11); - set_copro_access(venum_new_val); - - /* Enable FPEXC */ - *fp_orig_val = fmrx(FPEXC); - fp_new_val = *fp_orig_val | FPEXC_EN; - fmxr(FPEXC, fp_new_val); -} - -static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val) -{ - BUG_ON(preemptible()); - /* Restore FPEXC */ - fmxr(FPEXC, fp_orig_val); - isb(); - /* Restore CPACR */ - set_copro_access(venum_orig_val); -} - -static u32 krait_get_pmresrn_event(unsigned int region) -{ - static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0, - KRAIT_PMRESR1_GROUP0, - KRAIT_PMRESR2_GROUP0 }; - return pmresrn_table[region]; -} - -static void krait_evt_setup(int idx, u32 config_base) -{ - u32 val; - u32 mask; - u32 vval, fval; - unsigned int region = EVENT_REGION(config_base); - unsigned int group = EVENT_GROUP(config_base); - unsigned int code = EVENT_CODE(config_base); - unsigned int group_shift; - bool venum_event = EVENT_VENUM(config_base); - - group_shift = group * 8; - mask = 0xff << group_shift; - - /* Configure evtsel for the region and group */ - if (venum_event) - val = KRAIT_VPMRESR0_GROUP0; - else - val = krait_get_pmresrn_event(region); - val += group; - /* Mix in mode-exclusion bits */ - val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1); - armv7_pmnc_write_evtsel(idx, val); - - if (venum_event) { - venum_pre_pmresr(&vval, &fval); - val = venum_read_pmresr(); - val &= ~mask; - val |= code << group_shift; - val |= PMRESRn_EN; - venum_write_pmresr(val); - venum_post_pmresr(vval, fval); - } else { - val = krait_read_pmresrn(region); - val &= ~mask; - val |= code << group_shift; - val |= PMRESRn_EN; - krait_write_pmresrn(region, val); - } -} - -static u32 clear_pmresrn_group(u32 val, int group) -{ - u32 mask; - int group_shift; - - group_shift = group * 8; - mask = 0xff << group_shift; - val &= ~mask; - - /* Don't clear enable bit if entire region isn't disabled */ - if (val & ~PMRESRn_EN) - return val |= PMRESRn_EN; - - return 0; -} - -static void krait_clearpmu(u32 config_base) -{ - u32 val; - u32 vval, fval; - unsigned int region = EVENT_REGION(config_base); - unsigned int group = EVENT_GROUP(config_base); - bool venum_event = EVENT_VENUM(config_base); - - if (venum_event) { - venum_pre_pmresr(&vval, &fval); - val = venum_read_pmresr(); - val = clear_pmresrn_group(val, group); - venum_write_pmresr(val); - venum_post_pmresr(vval, fval); - } else { - val = krait_read_pmresrn(region); - val = clear_pmresrn_group(val, group); - krait_write_pmresrn(region, val); - } -} - -static void krait_pmu_disable_event(struct perf_event *event) -{ - unsigned long flags; - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - /* Disable counter and interrupt */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* Disable counter */ - armv7_pmnc_disable_counter(idx); - - /* - * Clear pmresr code (if destined for PMNx counters) - */ - if (hwc->config_base & KRAIT_EVENT_MASK) - krait_clearpmu(hwc->config_base); - - /* Disable interrupt for this counter */ - armv7_pmnc_disable_intens(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void krait_pmu_enable_event(struct perf_event *event) -{ - unsigned long flags; - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - /* - * Enable counter and interrupt, and set the counter to count - * the event that we're interested in. - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* Disable counter */ - armv7_pmnc_disable_counter(idx); - - /* - * Set event (if destined for PMNx counters) - * We set the event for the cycle counter because we - * have the ability to perform event filtering. - */ - if (hwc->config_base & KRAIT_EVENT_MASK) - krait_evt_setup(idx, hwc->config_base); - else - armv7_pmnc_write_evtsel(idx, hwc->config_base); - - /* Enable interrupt for this counter */ - armv7_pmnc_enable_intens(idx); - - /* Enable counter */ - armv7_pmnc_enable_counter(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void krait_pmu_reset(void *info) -{ - u32 vval, fval; - struct arm_pmu *cpu_pmu = info; - u32 idx, nb_cnt = cpu_pmu->num_events; - - armv7pmu_reset(info); - - /* Clear all pmresrs */ - krait_write_pmresrn(0, 0); - krait_write_pmresrn(1, 0); - krait_write_pmresrn(2, 0); - - venum_pre_pmresr(&vval, &fval); - venum_write_pmresr(0); - venum_post_pmresr(vval, fval); - - /* Reset PMxEVNCTCR to sane default */ - for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { - armv7_pmnc_select_counter(idx); - asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); - } - -} - -static int krait_event_to_bit(struct perf_event *event, unsigned int region, - unsigned int group) -{ - int bit; - struct hw_perf_event *hwc = &event->hw; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - - if (hwc->config_base & VENUM_EVENT) - bit = KRAIT_VPMRESR0_GROUP0; - else - bit = krait_get_pmresrn_event(region); - bit -= krait_get_pmresrn_event(0); - bit += group; - /* - * Lower bits are reserved for use by the counters (see - * armv7pmu_get_event_idx() for more info) - */ - bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1; - - return bit; -} - -/* - * We check for column exclusion constraints here. - * Two events cant use the same group within a pmresr register. - */ -static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - int idx; - int bit = -1; - struct hw_perf_event *hwc = &event->hw; - unsigned int region = EVENT_REGION(hwc->config_base); - unsigned int code = EVENT_CODE(hwc->config_base); - unsigned int group = EVENT_GROUP(hwc->config_base); - bool venum_event = EVENT_VENUM(hwc->config_base); - bool krait_event = EVENT_CPU(hwc->config_base); - - if (venum_event || krait_event) { - /* Ignore invalid events */ - if (group > 3 || region > 2) - return -EINVAL; - if (venum_event && (code & 0xe0)) - return -EINVAL; - - bit = krait_event_to_bit(event, region, group); - if (test_and_set_bit(bit, cpuc->used_mask)) - return -EAGAIN; - } - - idx = armv7pmu_get_event_idx(cpuc, event); - if (idx < 0 && bit >= 0) - clear_bit(bit, cpuc->used_mask); - - return idx; -} - -static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - int bit; - struct hw_perf_event *hwc = &event->hw; - unsigned int region = EVENT_REGION(hwc->config_base); - unsigned int group = EVENT_GROUP(hwc->config_base); - bool venum_event = EVENT_VENUM(hwc->config_base); - bool krait_event = EVENT_CPU(hwc->config_base); - - armv7pmu_clear_event_idx(cpuc, event); - if (venum_event || krait_event) { - bit = krait_event_to_bit(event, region, group); - clear_bit(bit, cpuc->used_mask); - } -} - -static int krait_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_krait"; - /* Some early versions of Krait don't support PC write events */ - if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node, - "qcom,no-pc-write")) - cpu_pmu->map_event = krait_map_event_no_branch; - else - cpu_pmu->map_event = krait_map_event; - cpu_pmu->set_event_filter = armv7pmu_set_event_filter; - cpu_pmu->reset = krait_pmu_reset; - cpu_pmu->enable = krait_pmu_enable_event; - cpu_pmu->disable = krait_pmu_disable_event; - cpu_pmu->get_event_idx = krait_pmu_get_event_idx; - cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx; - return armv7_probe_num_events(cpu_pmu); -} - -/* - * Scorpion Local Performance Monitor Register (LPMn) - * - * 31 30 24 16 8 0 - * +--------------------------------+ - * LPM0 | EN | CC | CC | CC | CC | N = 1, R = 0 - * +--------------------------------+ - * LPM1 | EN | CC | CC | CC | CC | N = 1, R = 1 - * +--------------------------------+ - * LPM2 | EN | CC | CC | CC | CC | N = 1, R = 2 - * +--------------------------------+ - * L2LPM | EN | CC | CC | CC | CC | N = 1, R = 3 - * +--------------------------------+ - * VLPM | EN | CC | CC | CC | CC | N = 2, R = ? - * +--------------------------------+ - * EN | G=3 | G=2 | G=1 | G=0 - * - * - * Event Encoding: - * - * hwc->config_base = 0xNRCCG - * - * N = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM) - * R = region register - * CC = class of events the group G is choosing from - * G = group or particular event - * - * Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2 - * - * A region (R) corresponds to a piece of the CPU (execution unit, instruction - * unit, etc.) while the event code (CC) corresponds to a particular class of - * events (interrupts for example). An event code is broken down into - * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for - * example). - */ - -static u32 scorpion_read_pmresrn(int n) -{ - u32 val; - - switch (n) { - case 0: - asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val)); - break; - case 1: - asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val)); - break; - case 2: - asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val)); - break; - case 3: - asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val)); - break; - default: - BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */ - } - - return val; -} - -static void scorpion_write_pmresrn(int n, u32 val) -{ - switch (n) { - case 0: - asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val)); - break; - case 1: - asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val)); - break; - case 2: - asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val)); - break; - case 3: - asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val)); - break; - default: - BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */ - } -} - -static u32 scorpion_get_pmresrn_event(unsigned int region) -{ - static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0, - SCORPION_LPM1_GROUP0, - SCORPION_LPM2_GROUP0, - SCORPION_L2LPM_GROUP0 }; - return pmresrn_table[region]; -} - -static void scorpion_evt_setup(int idx, u32 config_base) -{ - u32 val; - u32 mask; - u32 vval, fval; - unsigned int region = EVENT_REGION(config_base); - unsigned int group = EVENT_GROUP(config_base); - unsigned int code = EVENT_CODE(config_base); - unsigned int group_shift; - bool venum_event = EVENT_VENUM(config_base); - - group_shift = group * 8; - mask = 0xff << group_shift; - - /* Configure evtsel for the region and group */ - if (venum_event) - val = SCORPION_VLPM_GROUP0; - else - val = scorpion_get_pmresrn_event(region); - val += group; - /* Mix in mode-exclusion bits */ - val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1); - armv7_pmnc_write_evtsel(idx, val); - - asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); - - if (venum_event) { - venum_pre_pmresr(&vval, &fval); - val = venum_read_pmresr(); - val &= ~mask; - val |= code << group_shift; - val |= PMRESRn_EN; - venum_write_pmresr(val); - venum_post_pmresr(vval, fval); - } else { - val = scorpion_read_pmresrn(region); - val &= ~mask; - val |= code << group_shift; - val |= PMRESRn_EN; - scorpion_write_pmresrn(region, val); - } -} - -static void scorpion_clearpmu(u32 config_base) -{ - u32 val; - u32 vval, fval; - unsigned int region = EVENT_REGION(config_base); - unsigned int group = EVENT_GROUP(config_base); - bool venum_event = EVENT_VENUM(config_base); - - if (venum_event) { - venum_pre_pmresr(&vval, &fval); - val = venum_read_pmresr(); - val = clear_pmresrn_group(val, group); - venum_write_pmresr(val); - venum_post_pmresr(vval, fval); - } else { - val = scorpion_read_pmresrn(region); - val = clear_pmresrn_group(val, group); - scorpion_write_pmresrn(region, val); - } -} - -static void scorpion_pmu_disable_event(struct perf_event *event) -{ - unsigned long flags; - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - /* Disable counter and interrupt */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* Disable counter */ - armv7_pmnc_disable_counter(idx); - - /* - * Clear pmresr code (if destined for PMNx counters) - */ - if (hwc->config_base & KRAIT_EVENT_MASK) - scorpion_clearpmu(hwc->config_base); - - /* Disable interrupt for this counter */ - armv7_pmnc_disable_intens(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void scorpion_pmu_enable_event(struct perf_event *event) -{ - unsigned long flags; - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - /* - * Enable counter and interrupt, and set the counter to count - * the event that we're interested in. - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* Disable counter */ - armv7_pmnc_disable_counter(idx); - - /* - * Set event (if destined for PMNx counters) - * We don't set the event for the cycle counter because we - * don't have the ability to perform event filtering. - */ - if (hwc->config_base & KRAIT_EVENT_MASK) - scorpion_evt_setup(idx, hwc->config_base); - else if (idx != ARMV7_IDX_CYCLE_COUNTER) - armv7_pmnc_write_evtsel(idx, hwc->config_base); - - /* Enable interrupt for this counter */ - armv7_pmnc_enable_intens(idx); - - /* Enable counter */ - armv7_pmnc_enable_counter(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void scorpion_pmu_reset(void *info) -{ - u32 vval, fval; - struct arm_pmu *cpu_pmu = info; - u32 idx, nb_cnt = cpu_pmu->num_events; - - armv7pmu_reset(info); - - /* Clear all pmresrs */ - scorpion_write_pmresrn(0, 0); - scorpion_write_pmresrn(1, 0); - scorpion_write_pmresrn(2, 0); - scorpion_write_pmresrn(3, 0); - - venum_pre_pmresr(&vval, &fval); - venum_write_pmresr(0); - venum_post_pmresr(vval, fval); - - /* Reset PMxEVNCTCR to sane default */ - for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { - armv7_pmnc_select_counter(idx); - asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); - } -} - -static int scorpion_event_to_bit(struct perf_event *event, unsigned int region, - unsigned int group) -{ - int bit; - struct hw_perf_event *hwc = &event->hw; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - - if (hwc->config_base & VENUM_EVENT) - bit = SCORPION_VLPM_GROUP0; - else - bit = scorpion_get_pmresrn_event(region); - bit -= scorpion_get_pmresrn_event(0); - bit += group; - /* - * Lower bits are reserved for use by the counters (see - * armv7pmu_get_event_idx() for more info) - */ - bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1; - - return bit; -} - -/* - * We check for column exclusion constraints here. - * Two events cant use the same group within a pmresr register. - */ -static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - int idx; - int bit = -1; - struct hw_perf_event *hwc = &event->hw; - unsigned int region = EVENT_REGION(hwc->config_base); - unsigned int group = EVENT_GROUP(hwc->config_base); - bool venum_event = EVENT_VENUM(hwc->config_base); - bool scorpion_event = EVENT_CPU(hwc->config_base); - - if (venum_event || scorpion_event) { - /* Ignore invalid events */ - if (group > 3 || region > 3) - return -EINVAL; - - bit = scorpion_event_to_bit(event, region, group); - if (test_and_set_bit(bit, cpuc->used_mask)) - return -EAGAIN; - } - - idx = armv7pmu_get_event_idx(cpuc, event); - if (idx < 0 && bit >= 0) - clear_bit(bit, cpuc->used_mask); - - return idx; -} - -static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - int bit; - struct hw_perf_event *hwc = &event->hw; - unsigned int region = EVENT_REGION(hwc->config_base); - unsigned int group = EVENT_GROUP(hwc->config_base); - bool venum_event = EVENT_VENUM(hwc->config_base); - bool scorpion_event = EVENT_CPU(hwc->config_base); - - armv7pmu_clear_event_idx(cpuc, event); - if (venum_event || scorpion_event) { - bit = scorpion_event_to_bit(event, region, group); - clear_bit(bit, cpuc->used_mask); - } -} - -static int scorpion_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_scorpion"; - cpu_pmu->map_event = scorpion_map_event; - cpu_pmu->reset = scorpion_pmu_reset; - cpu_pmu->enable = scorpion_pmu_enable_event; - cpu_pmu->disable = scorpion_pmu_disable_event; - cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx; - cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx; - return armv7_probe_num_events(cpu_pmu); -} - -static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_scorpion_mp"; - cpu_pmu->map_event = scorpion_map_event; - cpu_pmu->reset = scorpion_pmu_reset; - cpu_pmu->enable = scorpion_pmu_enable_event; - cpu_pmu->disable = scorpion_pmu_disable_event; - cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx; - cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx; - return armv7_probe_num_events(cpu_pmu); -} - -static const struct of_device_id armv7_pmu_of_device_ids[] = { - {.compatible = "arm,cortex-a17-pmu", .data = armv7_a17_pmu_init}, - {.compatible = "arm,cortex-a15-pmu", .data = armv7_a15_pmu_init}, - {.compatible = "arm,cortex-a12-pmu", .data = armv7_a12_pmu_init}, - {.compatible = "arm,cortex-a9-pmu", .data = armv7_a9_pmu_init}, - {.compatible = "arm,cortex-a8-pmu", .data = armv7_a8_pmu_init}, - {.compatible = "arm,cortex-a7-pmu", .data = armv7_a7_pmu_init}, - {.compatible = "arm,cortex-a5-pmu", .data = armv7_a5_pmu_init}, - {.compatible = "qcom,krait-pmu", .data = krait_pmu_init}, - {.compatible = "qcom,scorpion-pmu", .data = scorpion_pmu_init}, - {.compatible = "qcom,scorpion-mp-pmu", .data = scorpion_mp_pmu_init}, - {}, -}; - -static const struct pmu_probe_info armv7_pmu_probe_table[] = { - ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A8, armv7_a8_pmu_init), - ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A9, armv7_a9_pmu_init), - { /* sentinel value */ } -}; - - -static int armv7_pmu_device_probe(struct platform_device *pdev) -{ - return arm_pmu_device_probe(pdev, armv7_pmu_of_device_ids, - armv7_pmu_probe_table); -} - -static struct platform_driver armv7_pmu_driver = { - .driver = { - .name = "armv7-pmu", - .of_match_table = armv7_pmu_of_device_ids, - .suppress_bind_attrs = true, - }, - .probe = armv7_pmu_device_probe, -}; - -builtin_platform_driver(armv7_pmu_driver); -#endif /* CONFIG_CPU_V7 */ diff --git a/arch/arm/kernel/perf_event_xscale.c b/arch/arm/kernel/perf_event_xscale.c deleted file mode 100644 index f6cdcacfb96d..000000000000 --- a/arch/arm/kernel/perf_event_xscale.c +++ /dev/null @@ -1,776 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * ARMv5 [xscale] Performance counter handling code. - * - * Copyright (C) 2010, ARM Ltd., Will Deacon <will.deacon@arm.com> - * - * Based on the previous xscale OProfile code. - * - * There are two variants of the xscale PMU that we support: - * - xscale1pmu: 2 event counters and a cycle counter - * - xscale2pmu: 4 event counters and a cycle counter - * The two variants share event definitions, but have different - * PMU structures. - */ - -#ifdef CONFIG_CPU_XSCALE - -#include <asm/cputype.h> -#include <asm/irq_regs.h> - -#include <linux/of.h> -#include <linux/perf/arm_pmu.h> -#include <linux/platform_device.h> - -enum xscale_perf_types { - XSCALE_PERFCTR_ICACHE_MISS = 0x00, - XSCALE_PERFCTR_ICACHE_NO_DELIVER = 0x01, - XSCALE_PERFCTR_DATA_STALL = 0x02, - XSCALE_PERFCTR_ITLB_MISS = 0x03, - XSCALE_PERFCTR_DTLB_MISS = 0x04, - XSCALE_PERFCTR_BRANCH = 0x05, - XSCALE_PERFCTR_BRANCH_MISS = 0x06, - XSCALE_PERFCTR_INSTRUCTION = 0x07, - XSCALE_PERFCTR_DCACHE_FULL_STALL = 0x08, - XSCALE_PERFCTR_DCACHE_FULL_STALL_CONTIG = 0x09, - XSCALE_PERFCTR_DCACHE_ACCESS = 0x0A, - XSCALE_PERFCTR_DCACHE_MISS = 0x0B, - XSCALE_PERFCTR_DCACHE_WRITE_BACK = 0x0C, - XSCALE_PERFCTR_PC_CHANGED = 0x0D, - XSCALE_PERFCTR_BCU_REQUEST = 0x10, - XSCALE_PERFCTR_BCU_FULL = 0x11, - XSCALE_PERFCTR_BCU_DRAIN = 0x12, - XSCALE_PERFCTR_BCU_ECC_NO_ELOG = 0x14, - XSCALE_PERFCTR_BCU_1_BIT_ERR = 0x15, - XSCALE_PERFCTR_RMW = 0x16, - /* XSCALE_PERFCTR_CCNT is not hardware defined */ - XSCALE_PERFCTR_CCNT = 0xFE, - XSCALE_PERFCTR_UNUSED = 0xFF, -}; - -enum xscale_counters { - XSCALE_CYCLE_COUNTER = 0, - XSCALE_COUNTER0, - XSCALE_COUNTER1, - XSCALE_COUNTER2, - XSCALE_COUNTER3, -}; - -static const unsigned xscale_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = XSCALE_PERFCTR_CCNT, - [PERF_COUNT_HW_INSTRUCTIONS] = XSCALE_PERFCTR_INSTRUCTION, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = XSCALE_PERFCTR_BRANCH, - [PERF_COUNT_HW_BRANCH_MISSES] = XSCALE_PERFCTR_BRANCH_MISS, - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = XSCALE_PERFCTR_ICACHE_NO_DELIVER, -}; - -static const unsigned xscale_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = XSCALE_PERFCTR_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = XSCALE_PERFCTR_DCACHE_MISS, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = XSCALE_PERFCTR_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = XSCALE_PERFCTR_DCACHE_MISS, - - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = XSCALE_PERFCTR_ICACHE_MISS, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = XSCALE_PERFCTR_DTLB_MISS, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = XSCALE_PERFCTR_DTLB_MISS, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = XSCALE_PERFCTR_ITLB_MISS, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = XSCALE_PERFCTR_ITLB_MISS, -}; - -#define XSCALE_PMU_ENABLE 0x001 -#define XSCALE_PMN_RESET 0x002 -#define XSCALE_CCNT_RESET 0x004 -#define XSCALE_PMU_RESET (CCNT_RESET | PMN_RESET) -#define XSCALE_PMU_CNT64 0x008 - -#define XSCALE1_OVERFLOWED_MASK 0x700 -#define XSCALE1_CCOUNT_OVERFLOW 0x400 -#define XSCALE1_COUNT0_OVERFLOW 0x100 -#define XSCALE1_COUNT1_OVERFLOW 0x200 -#define XSCALE1_CCOUNT_INT_EN 0x040 -#define XSCALE1_COUNT0_INT_EN 0x010 -#define XSCALE1_COUNT1_INT_EN 0x020 -#define XSCALE1_COUNT0_EVT_SHFT 12 -#define XSCALE1_COUNT0_EVT_MASK (0xff << XSCALE1_COUNT0_EVT_SHFT) -#define XSCALE1_COUNT1_EVT_SHFT 20 -#define XSCALE1_COUNT1_EVT_MASK (0xff << XSCALE1_COUNT1_EVT_SHFT) - -static inline u32 -xscale1pmu_read_pmnc(void) -{ - u32 val; - asm volatile("mrc p14, 0, %0, c0, c0, 0" : "=r" (val)); - return val; -} - -static inline void -xscale1pmu_write_pmnc(u32 val) -{ - /* upper 4bits and 7, 11 are write-as-0 */ - val &= 0xffff77f; - asm volatile("mcr p14, 0, %0, c0, c0, 0" : : "r" (val)); -} - -static inline int -xscale1_pmnc_counter_has_overflowed(unsigned long pmnc, - enum xscale_counters counter) -{ - int ret = 0; - - switch (counter) { - case XSCALE_CYCLE_COUNTER: - ret = pmnc & XSCALE1_CCOUNT_OVERFLOW; - break; - case XSCALE_COUNTER0: - ret = pmnc & XSCALE1_COUNT0_OVERFLOW; - break; - case XSCALE_COUNTER1: - ret = pmnc & XSCALE1_COUNT1_OVERFLOW; - break; - default: - WARN_ONCE(1, "invalid counter number (%d)\n", counter); - } - - return ret; -} - -static irqreturn_t -xscale1pmu_handle_irq(struct arm_pmu *cpu_pmu) -{ - unsigned long pmnc; - struct perf_sample_data data; - struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); - struct pt_regs *regs; - int idx; - - /* - * NOTE: there's an A stepping erratum that states if an overflow - * bit already exists and another occurs, the previous - * Overflow bit gets cleared. There's no workaround. - * Fixed in B stepping or later. - */ - pmnc = xscale1pmu_read_pmnc(); - - /* - * Write the value back to clear the overflow flags. Overflow - * flags remain in pmnc for use below. We also disable the PMU - * while we process the interrupt. - */ - xscale1pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE); - - if (!(pmnc & XSCALE1_OVERFLOWED_MASK)) - return IRQ_NONE; - - regs = get_irq_regs(); - - for (idx = 0; idx < cpu_pmu->num_events; ++idx) { - struct perf_event *event = cpuc->events[idx]; - struct hw_perf_event *hwc; - - if (!event) - continue; - - if (!xscale1_pmnc_counter_has_overflowed(pmnc, idx)) - continue; - - hwc = &event->hw; - armpmu_event_update(event); - perf_sample_data_init(&data, 0, hwc->last_period); - if (!armpmu_event_set_period(event)) - continue; - - if (perf_event_overflow(event, &data, regs)) - cpu_pmu->disable(event); - } - - irq_work_run(); - - /* - * Re-enable the PMU. - */ - pmnc = xscale1pmu_read_pmnc() | XSCALE_PMU_ENABLE; - xscale1pmu_write_pmnc(pmnc); - - return IRQ_HANDLED; -} - -static void xscale1pmu_enable_event(struct perf_event *event) -{ - unsigned long val, mask, evt, flags; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - switch (idx) { - case XSCALE_CYCLE_COUNTER: - mask = 0; - evt = XSCALE1_CCOUNT_INT_EN; - break; - case XSCALE_COUNTER0: - mask = XSCALE1_COUNT0_EVT_MASK; - evt = (hwc->config_base << XSCALE1_COUNT0_EVT_SHFT) | - XSCALE1_COUNT0_INT_EN; - break; - case XSCALE_COUNTER1: - mask = XSCALE1_COUNT1_EVT_MASK; - evt = (hwc->config_base << XSCALE1_COUNT1_EVT_SHFT) | - XSCALE1_COUNT1_INT_EN; - break; - default: - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = xscale1pmu_read_pmnc(); - val &= ~mask; - val |= evt; - xscale1pmu_write_pmnc(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void xscale1pmu_disable_event(struct perf_event *event) -{ - unsigned long val, mask, evt, flags; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - switch (idx) { - case XSCALE_CYCLE_COUNTER: - mask = XSCALE1_CCOUNT_INT_EN; - evt = 0; - break; - case XSCALE_COUNTER0: - mask = XSCALE1_COUNT0_INT_EN | XSCALE1_COUNT0_EVT_MASK; - evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT0_EVT_SHFT; - break; - case XSCALE_COUNTER1: - mask = XSCALE1_COUNT1_INT_EN | XSCALE1_COUNT1_EVT_MASK; - evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT1_EVT_SHFT; - break; - default: - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = xscale1pmu_read_pmnc(); - val &= ~mask; - val |= evt; - xscale1pmu_write_pmnc(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static int -xscale1pmu_get_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - struct hw_perf_event *hwc = &event->hw; - if (XSCALE_PERFCTR_CCNT == hwc->config_base) { - if (test_and_set_bit(XSCALE_CYCLE_COUNTER, cpuc->used_mask)) - return -EAGAIN; - - return XSCALE_CYCLE_COUNTER; - } else { - if (!test_and_set_bit(XSCALE_COUNTER1, cpuc->used_mask)) - return XSCALE_COUNTER1; - - if (!test_and_set_bit(XSCALE_COUNTER0, cpuc->used_mask)) - return XSCALE_COUNTER0; - - return -EAGAIN; - } -} - -static void xscalepmu_clear_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - clear_bit(event->hw.idx, cpuc->used_mask); -} - -static void xscale1pmu_start(struct arm_pmu *cpu_pmu) -{ - unsigned long flags, val; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = xscale1pmu_read_pmnc(); - val |= XSCALE_PMU_ENABLE; - xscale1pmu_write_pmnc(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void xscale1pmu_stop(struct arm_pmu *cpu_pmu) -{ - unsigned long flags, val; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = xscale1pmu_read_pmnc(); - val &= ~XSCALE_PMU_ENABLE; - xscale1pmu_write_pmnc(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static inline u64 xscale1pmu_read_counter(struct perf_event *event) -{ - struct hw_perf_event *hwc = &event->hw; - int counter = hwc->idx; - u32 val = 0; - - switch (counter) { - case XSCALE_CYCLE_COUNTER: - asm volatile("mrc p14, 0, %0, c1, c0, 0" : "=r" (val)); - break; - case XSCALE_COUNTER0: - asm volatile("mrc p14, 0, %0, c2, c0, 0" : "=r" (val)); - break; - case XSCALE_COUNTER1: - asm volatile("mrc p14, 0, %0, c3, c0, 0" : "=r" (val)); - break; - } - - return val; -} - -static inline void xscale1pmu_write_counter(struct perf_event *event, u64 val) -{ - struct hw_perf_event *hwc = &event->hw; - int counter = hwc->idx; - - switch (counter) { - case XSCALE_CYCLE_COUNTER: - asm volatile("mcr p14, 0, %0, c1, c0, 0" : : "r" (val)); - break; - case XSCALE_COUNTER0: - asm volatile("mcr p14, 0, %0, c2, c0, 0" : : "r" (val)); - break; - case XSCALE_COUNTER1: - asm volatile("mcr p14, 0, %0, c3, c0, 0" : : "r" (val)); - break; - } -} - -static int xscale_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &xscale_perf_map, - &xscale_perf_cache_map, 0xFF); -} - -static int xscale1pmu_init(struct arm_pmu *cpu_pmu) -{ - cpu_pmu->name = "armv5_xscale1"; - cpu_pmu->handle_irq = xscale1pmu_handle_irq; - cpu_pmu->enable = xscale1pmu_enable_event; - cpu_pmu->disable = xscale1pmu_disable_event; - cpu_pmu->read_counter = xscale1pmu_read_counter; - cpu_pmu->write_counter = xscale1pmu_write_counter; - cpu_pmu->get_event_idx = xscale1pmu_get_event_idx; - cpu_pmu->clear_event_idx = xscalepmu_clear_event_idx; - cpu_pmu->start = xscale1pmu_start; - cpu_pmu->stop = xscale1pmu_stop; - cpu_pmu->map_event = xscale_map_event; - cpu_pmu->num_events = 3; - - return 0; -} - -#define XSCALE2_OVERFLOWED_MASK 0x01f -#define XSCALE2_CCOUNT_OVERFLOW 0x001 -#define XSCALE2_COUNT0_OVERFLOW 0x002 -#define XSCALE2_COUNT1_OVERFLOW 0x004 -#define XSCALE2_COUNT2_OVERFLOW 0x008 -#define XSCALE2_COUNT3_OVERFLOW 0x010 -#define XSCALE2_CCOUNT_INT_EN 0x001 -#define XSCALE2_COUNT0_INT_EN 0x002 -#define XSCALE2_COUNT1_INT_EN 0x004 -#define XSCALE2_COUNT2_INT_EN 0x008 -#define XSCALE2_COUNT3_INT_EN 0x010 -#define XSCALE2_COUNT0_EVT_SHFT 0 -#define XSCALE2_COUNT0_EVT_MASK (0xff << XSCALE2_COUNT0_EVT_SHFT) -#define XSCALE2_COUNT1_EVT_SHFT 8 -#define XSCALE2_COUNT1_EVT_MASK (0xff << XSCALE2_COUNT1_EVT_SHFT) -#define XSCALE2_COUNT2_EVT_SHFT 16 -#define XSCALE2_COUNT2_EVT_MASK (0xff << XSCALE2_COUNT2_EVT_SHFT) -#define XSCALE2_COUNT3_EVT_SHFT 24 -#define XSCALE2_COUNT3_EVT_MASK (0xff << XSCALE2_COUNT3_EVT_SHFT) - -static inline u32 -xscale2pmu_read_pmnc(void) -{ - u32 val; - asm volatile("mrc p14, 0, %0, c0, c1, 0" : "=r" (val)); - /* bits 1-2 and 4-23 are read-unpredictable */ - return val & 0xff000009; -} - -static inline void -xscale2pmu_write_pmnc(u32 val) -{ - /* bits 4-23 are write-as-0, 24-31 are write ignored */ - val &= 0xf; - asm volatile("mcr p14, 0, %0, c0, c1, 0" : : "r" (val)); -} - -static inline u32 -xscale2pmu_read_overflow_flags(void) -{ - u32 val; - asm volatile("mrc p14, 0, %0, c5, c1, 0" : "=r" (val)); - return val; -} - -static inline void -xscale2pmu_write_overflow_flags(u32 val) -{ - asm volatile("mcr p14, 0, %0, c5, c1, 0" : : "r" (val)); -} - -static inline u32 -xscale2pmu_read_event_select(void) -{ - u32 val; - asm volatile("mrc p14, 0, %0, c8, c1, 0" : "=r" (val)); - return val; -} - -static inline void -xscale2pmu_write_event_select(u32 val) -{ - asm volatile("mcr p14, 0, %0, c8, c1, 0" : : "r"(val)); -} - -static inline u32 -xscale2pmu_read_int_enable(void) -{ - u32 val; - asm volatile("mrc p14, 0, %0, c4, c1, 0" : "=r" (val)); - return val; -} - -static void -xscale2pmu_write_int_enable(u32 val) -{ - asm volatile("mcr p14, 0, %0, c4, c1, 0" : : "r" (val)); -} - -static inline int -xscale2_pmnc_counter_has_overflowed(unsigned long of_flags, - enum xscale_counters counter) -{ - int ret = 0; - - switch (counter) { - case XSCALE_CYCLE_COUNTER: - ret = of_flags & XSCALE2_CCOUNT_OVERFLOW; - break; - case XSCALE_COUNTER0: - ret = of_flags & XSCALE2_COUNT0_OVERFLOW; - break; - case XSCALE_COUNTER1: - ret = of_flags & XSCALE2_COUNT1_OVERFLOW; - break; - case XSCALE_COUNTER2: - ret = of_flags & XSCALE2_COUNT2_OVERFLOW; - break; - case XSCALE_COUNTER3: - ret = of_flags & XSCALE2_COUNT3_OVERFLOW; - break; - default: - WARN_ONCE(1, "invalid counter number (%d)\n", counter); - } - - return ret; -} - -static irqreturn_t -xscale2pmu_handle_irq(struct arm_pmu *cpu_pmu) -{ - unsigned long pmnc, of_flags; - struct perf_sample_data data; - struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); - struct pt_regs *regs; - int idx; - - /* Disable the PMU. */ - pmnc = xscale2pmu_read_pmnc(); - xscale2pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE); - - /* Check the overflow flag register. */ - of_flags = xscale2pmu_read_overflow_flags(); - if (!(of_flags & XSCALE2_OVERFLOWED_MASK)) - return IRQ_NONE; - - /* Clear the overflow bits. */ - xscale2pmu_write_overflow_flags(of_flags); - - regs = get_irq_regs(); - - for (idx = 0; idx < cpu_pmu->num_events; ++idx) { - struct perf_event *event = cpuc->events[idx]; - struct hw_perf_event *hwc; - - if (!event) - continue; - - if (!xscale2_pmnc_counter_has_overflowed(of_flags, idx)) - continue; - - hwc = &event->hw; - armpmu_event_update(event); - perf_sample_data_init(&data, 0, hwc->last_period); - if (!armpmu_event_set_period(event)) - continue; - - if (perf_event_overflow(event, &data, regs)) - cpu_pmu->disable(event); - } - - irq_work_run(); - - /* - * Re-enable the PMU. - */ - pmnc = xscale2pmu_read_pmnc() | XSCALE_PMU_ENABLE; - xscale2pmu_write_pmnc(pmnc); - - return IRQ_HANDLED; -} - -static void xscale2pmu_enable_event(struct perf_event *event) -{ - unsigned long flags, ien, evtsel; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - ien = xscale2pmu_read_int_enable(); - evtsel = xscale2pmu_read_event_select(); - - switch (idx) { - case XSCALE_CYCLE_COUNTER: - ien |= XSCALE2_CCOUNT_INT_EN; - break; - case XSCALE_COUNTER0: - ien |= XSCALE2_COUNT0_INT_EN; - evtsel &= ~XSCALE2_COUNT0_EVT_MASK; - evtsel |= hwc->config_base << XSCALE2_COUNT0_EVT_SHFT; - break; - case XSCALE_COUNTER1: - ien |= XSCALE2_COUNT1_INT_EN; - evtsel &= ~XSCALE2_COUNT1_EVT_MASK; - evtsel |= hwc->config_base << XSCALE2_COUNT1_EVT_SHFT; - break; - case XSCALE_COUNTER2: - ien |= XSCALE2_COUNT2_INT_EN; - evtsel &= ~XSCALE2_COUNT2_EVT_MASK; - evtsel |= hwc->config_base << XSCALE2_COUNT2_EVT_SHFT; - break; - case XSCALE_COUNTER3: - ien |= XSCALE2_COUNT3_INT_EN; - evtsel &= ~XSCALE2_COUNT3_EVT_MASK; - evtsel |= hwc->config_base << XSCALE2_COUNT3_EVT_SHFT; - break; - default: - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - xscale2pmu_write_event_select(evtsel); - xscale2pmu_write_int_enable(ien); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void xscale2pmu_disable_event(struct perf_event *event) -{ - unsigned long flags, ien, evtsel, of_flags; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - ien = xscale2pmu_read_int_enable(); - evtsel = xscale2pmu_read_event_select(); - - switch (idx) { - case XSCALE_CYCLE_COUNTER: - ien &= ~XSCALE2_CCOUNT_INT_EN; - of_flags = XSCALE2_CCOUNT_OVERFLOW; - break; - case XSCALE_COUNTER0: - ien &= ~XSCALE2_COUNT0_INT_EN; - evtsel &= ~XSCALE2_COUNT0_EVT_MASK; - evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT0_EVT_SHFT; - of_flags = XSCALE2_COUNT0_OVERFLOW; - break; - case XSCALE_COUNTER1: - ien &= ~XSCALE2_COUNT1_INT_EN; - evtsel &= ~XSCALE2_COUNT1_EVT_MASK; - evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT1_EVT_SHFT; - of_flags = XSCALE2_COUNT1_OVERFLOW; - break; - case XSCALE_COUNTER2: - ien &= ~XSCALE2_COUNT2_INT_EN; - evtsel &= ~XSCALE2_COUNT2_EVT_MASK; - evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT2_EVT_SHFT; - of_flags = XSCALE2_COUNT2_OVERFLOW; - break; - case XSCALE_COUNTER3: - ien &= ~XSCALE2_COUNT3_INT_EN; - evtsel &= ~XSCALE2_COUNT3_EVT_MASK; - evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT3_EVT_SHFT; - of_flags = XSCALE2_COUNT3_OVERFLOW; - break; - default: - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - xscale2pmu_write_event_select(evtsel); - xscale2pmu_write_int_enable(ien); - xscale2pmu_write_overflow_flags(of_flags); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static int -xscale2pmu_get_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - int idx = xscale1pmu_get_event_idx(cpuc, event); - if (idx >= 0) - goto out; - - if (!test_and_set_bit(XSCALE_COUNTER3, cpuc->used_mask)) - idx = XSCALE_COUNTER3; - else if (!test_and_set_bit(XSCALE_COUNTER2, cpuc->used_mask)) - idx = XSCALE_COUNTER2; -out: - return idx; -} - -static void xscale2pmu_start(struct arm_pmu *cpu_pmu) -{ - unsigned long flags, val; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = xscale2pmu_read_pmnc() & ~XSCALE_PMU_CNT64; - val |= XSCALE_PMU_ENABLE; - xscale2pmu_write_pmnc(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void xscale2pmu_stop(struct arm_pmu *cpu_pmu) -{ - unsigned long flags, val; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = xscale2pmu_read_pmnc(); - val &= ~XSCALE_PMU_ENABLE; - xscale2pmu_write_pmnc(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static inline u64 xscale2pmu_read_counter(struct perf_event *event) -{ - struct hw_perf_event *hwc = &event->hw; - int counter = hwc->idx; - u32 val = 0; - - switch (counter) { - case XSCALE_CYCLE_COUNTER: - asm volatile("mrc p14, 0, %0, c1, c1, 0" : "=r" (val)); - break; - case XSCALE_COUNTER0: - asm volatile("mrc p14, 0, %0, c0, c2, 0" : "=r" (val)); - break; - case XSCALE_COUNTER1: - asm volatile("mrc p14, 0, %0, c1, c2, 0" : "=r" (val)); - break; - case XSCALE_COUNTER2: - asm volatile("mrc p14, 0, %0, c2, c2, 0" : "=r" (val)); - break; - case XSCALE_COUNTER3: - asm volatile("mrc p14, 0, %0, c3, c2, 0" : "=r" (val)); - break; - } - - return val; -} - -static inline void xscale2pmu_write_counter(struct perf_event *event, u64 val) -{ - struct hw_perf_event *hwc = &event->hw; - int counter = hwc->idx; - - switch (counter) { - case XSCALE_CYCLE_COUNTER: - asm volatile("mcr p14, 0, %0, c1, c1, 0" : : "r" (val)); - break; - case XSCALE_COUNTER0: - asm volatile("mcr p14, 0, %0, c0, c2, 0" : : "r" (val)); - break; - case XSCALE_COUNTER1: - asm volatile("mcr p14, 0, %0, c1, c2, 0" : : "r" (val)); - break; - case XSCALE_COUNTER2: - asm volatile("mcr p14, 0, %0, c2, c2, 0" : : "r" (val)); - break; - case XSCALE_COUNTER3: - asm volatile("mcr p14, 0, %0, c3, c2, 0" : : "r" (val)); - break; - } -} - -static int xscale2pmu_init(struct arm_pmu *cpu_pmu) -{ - cpu_pmu->name = "armv5_xscale2"; - cpu_pmu->handle_irq = xscale2pmu_handle_irq; - cpu_pmu->enable = xscale2pmu_enable_event; - cpu_pmu->disable = xscale2pmu_disable_event; - cpu_pmu->read_counter = xscale2pmu_read_counter; - cpu_pmu->write_counter = xscale2pmu_write_counter; - cpu_pmu->get_event_idx = xscale2pmu_get_event_idx; - cpu_pmu->clear_event_idx = xscalepmu_clear_event_idx; - cpu_pmu->start = xscale2pmu_start; - cpu_pmu->stop = xscale2pmu_stop; - cpu_pmu->map_event = xscale_map_event; - cpu_pmu->num_events = 5; - - return 0; -} - -static const struct pmu_probe_info xscale_pmu_probe_table[] = { - XSCALE_PMU_PROBE(ARM_CPU_XSCALE_ARCH_V1, xscale1pmu_init), - XSCALE_PMU_PROBE(ARM_CPU_XSCALE_ARCH_V2, xscale2pmu_init), - { /* sentinel value */ } -}; - -static int xscale_pmu_device_probe(struct platform_device *pdev) -{ - return arm_pmu_device_probe(pdev, NULL, xscale_pmu_probe_table); -} - -static struct platform_driver xscale_pmu_driver = { - .driver = { - .name = "xscale-pmu", - }, - .probe = xscale_pmu_device_probe, -}; - -builtin_platform_driver(xscale_pmu_driver); -#endif /* CONFIG_CPU_XSCALE */ diff --git a/arch/arm/kernel/pj4-cp0.c b/arch/arm/kernel/pj4-cp0.c deleted file mode 100644 index 1d1fb22f44f3..000000000000 --- a/arch/arm/kernel/pj4-cp0.c +++ /dev/null @@ -1,134 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * linux/arch/arm/kernel/pj4-cp0.c - * - * PJ4 iWMMXt coprocessor context switching and handling - * - * Copyright (c) 2010 Marvell International Inc. - */ - -#include <linux/types.h> -#include <linux/kernel.h> -#include <linux/signal.h> -#include <linux/sched.h> -#include <linux/init.h> -#include <linux/io.h> -#include <asm/thread_notify.h> -#include <asm/cputype.h> - -static int iwmmxt_do(struct notifier_block *self, unsigned long cmd, void *t) -{ - struct thread_info *thread = t; - - switch (cmd) { - case THREAD_NOTIFY_FLUSH: - /* - * flush_thread() zeroes thread->fpstate, so no need - * to do anything here. - * - * FALLTHROUGH: Ensure we don't try to overwrite our newly - * initialised state information on the first fault. - */ - - case THREAD_NOTIFY_EXIT: - iwmmxt_task_release(thread); - break; - - case THREAD_NOTIFY_SWITCH: - iwmmxt_task_switch(thread); - break; - } - - return NOTIFY_DONE; -} - -static struct notifier_block __maybe_unused iwmmxt_notifier_block = { - .notifier_call = iwmmxt_do, -}; - - -static u32 __init pj4_cp_access_read(void) -{ - u32 value; - - __asm__ __volatile__ ( - "mrc p15, 0, %0, c1, c0, 2\n\t" - : "=r" (value)); - return value; -} - -static void __init pj4_cp_access_write(u32 value) -{ - u32 temp; - - __asm__ __volatile__ ( - "mcr p15, 0, %1, c1, c0, 2\n\t" -#ifdef CONFIG_THUMB2_KERNEL - "isb\n\t" -#else - "mrc p15, 0, %0, c1, c0, 2\n\t" - "mov %0, %0\n\t" - "sub pc, pc, #4\n\t" -#endif - : "=r" (temp) : "r" (value)); -} - -static int __init pj4_get_iwmmxt_version(void) -{ - u32 cp_access, wcid; - - cp_access = pj4_cp_access_read(); - pj4_cp_access_write(cp_access | 0xf); - - /* check if coprocessor 0 and 1 are available */ - if ((pj4_cp_access_read() & 0xf) != 0xf) { - pj4_cp_access_write(cp_access); - return -ENODEV; - } - - /* read iWMMXt coprocessor id register p1, c0 */ - __asm__ __volatile__ ("mrc p1, 0, %0, c0, c0, 0\n" : "=r" (wcid)); - - pj4_cp_access_write(cp_access); - - /* iWMMXt v1 */ - if ((wcid & 0xffffff00) == 0x56051000) - return 1; - /* iWMMXt v2 */ - if ((wcid & 0xffffff00) == 0x56052000) - return 2; - - return -EINVAL; -} - -/* - * Disable CP0/CP1 on boot, and let call_fpe() and the iWMMXt lazy - * switch code handle iWMMXt context switching. - */ -static int __init pj4_cp0_init(void) -{ - u32 __maybe_unused cp_access; - int vers; - - if (!cpu_is_pj4()) - return 0; - - vers = pj4_get_iwmmxt_version(); - if (vers < 0) - return 0; - -#ifndef CONFIG_IWMMXT - pr_info("PJ4 iWMMXt coprocessor detected, but kernel support is missing.\n"); -#else - cp_access = pj4_cp_access_read() & ~0xf; - pj4_cp_access_write(cp_access); - - pr_info("PJ4 iWMMXt v%d coprocessor enabled.\n", vers); - elf_hwcap |= HWCAP_IWMMXT; - thread_register_notifier(&iwmmxt_notifier_block); -#endif - - return 0; -} - -late_initcall(pj4_cp0_init); diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index f811733a8fc5..d7aa95225c70 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -78,7 +78,6 @@ void arch_cpu_idle(void) arm_pm_idle(); else cpu_do_idle(); - raw_local_irq_enable(); } void arch_cpu_idle_prepare(void) @@ -223,7 +222,6 @@ void flush_thread(void) flush_ptrace_hw_breakpoint(tsk); - memset(thread->used_cp, 0, sizeof(thread->used_cp)); memset(&tsk->thread.debug, 0, sizeof(struct debug_info)); memset(&thread->fpstate, 0, sizeof(union fp_state)); @@ -236,7 +234,7 @@ asmlinkage void ret_from_fork(void) __asm__("ret_from_fork"); int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) { - unsigned long clone_flags = args->flags; + u64 clone_flags = args->flags; unsigned long stack_start = args->stack; unsigned long tls = args->tls; struct thread_info *thread = task_thread_info(p); @@ -316,7 +314,7 @@ static int __init gate_vma_init(void) gate_vma.vm_page_prot = PAGE_READONLY_EXEC; gate_vma.vm_start = 0xffff0000; gate_vma.vm_end = 0xffff0000 + PAGE_SIZE; - gate_vma.vm_flags = VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYEXEC; + vm_flags_init(&gate_vma, VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYEXEC); return 0; } arch_initcall(gate_vma_init); diff --git a/arch/arm/kernel/psci_smp.c b/arch/arm/kernel/psci_smp.c index d4392e177484..3bb0c4dcfc5c 100644 --- a/arch/arm/kernel/psci_smp.c +++ b/arch/arm/kernel/psci_smp.c @@ -45,8 +45,15 @@ extern void secondary_startup(void); static int psci_boot_secondary(unsigned int cpu, struct task_struct *idle) { if (psci_ops.cpu_on) +#ifdef CONFIG_XIP_KERNEL + return psci_ops.cpu_on(cpu_logical_map(cpu), + ((phys_addr_t)(&secondary_startup) + - XIP_VIRT_ADDR(CONFIG_XIP_PHYS_ADDR) + + CONFIG_XIP_PHYS_ADDR)); +#else return psci_ops.cpu_on(cpu_logical_map(cpu), virt_to_idmap(&secondary_startup)); +#endif return -ENODEV; } diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c index 2d8e2516906b..7951b2c06fec 100644 --- a/arch/arm/kernel/ptrace.c +++ b/arch/arm/kernel/ptrace.c @@ -584,8 +584,6 @@ static int fpa_set(struct task_struct *target, { struct thread_info *thread = task_thread_info(target); - thread->used_cp[1] = thread->used_cp[2] = 1; - return user_regset_copyin(&pos, &count, &kbuf, &ubuf, &thread->fpstate, 0, sizeof(struct user_fp)); @@ -679,7 +677,7 @@ enum arm_regset { static const struct user_regset arm_regsets[] = { [REGSET_GPR] = { - .core_note_type = NT_PRSTATUS, + USER_REGSET_NOTE_TYPE(PRSTATUS), .n = ELF_NGREG, .size = sizeof(u32), .align = sizeof(u32), @@ -691,7 +689,7 @@ static const struct user_regset arm_regsets[] = { * For the FPA regs in fpstate, the real fields are a mixture * of sizes, so pretend that the registers are word-sized: */ - .core_note_type = NT_PRFPREG, + USER_REGSET_NOTE_TYPE(PRFPREG), .n = sizeof(struct user_fp) / sizeof(u32), .size = sizeof(u32), .align = sizeof(u32), @@ -704,7 +702,7 @@ static const struct user_regset arm_regsets[] = { * Pretend that the VFP regs are word-sized, since the FPSCR is * a single word dangling at the end of struct user_vfp: */ - .core_note_type = NT_ARM_VFP, + USER_REGSET_NOTE_TYPE(ARM_VFP), .n = ARM_VFPREGS_SIZE / sizeof(u32), .size = sizeof(u32), .align = sizeof(u32), @@ -783,8 +781,9 @@ long arch_ptrace(struct task_struct *child, long request, break; case PTRACE_SET_SYSCALL: - task_thread_info(child)->abi_syscall = data & - __NR_SYSCALL_MASK; + if (data != -1) + data &= __NR_SYSCALL_MASK; + task_thread_info(child)->abi_syscall = data; ret = 0; break; diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 75cd4699e7b3..0bfd66c7ada0 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -15,10 +15,10 @@ #include <linux/console.h> #include <linux/seq_file.h> #include <linux/screen_info.h> -#include <linux/of_platform.h> #include <linux/init.h> #include <linux/kexec.h> #include <linux/libfdt.h> +#include <linux/of.h> #include <linux/of_fdt.h> #include <linux/cpu.h> #include <linux/interrupt.h> @@ -76,13 +76,6 @@ static int __init fpe_setup(char *line) __setup("fpe=", fpe_setup); #endif -extern void init_default_cache_policy(unsigned long); -extern void paging_init(const struct machine_desc *desc); -extern void early_mm_init(const struct machine_desc *); -extern void adjust_lowmem_bounds(void); -extern enum reboot_mode reboot_mode; -extern void setup_dma_zone(const struct machine_desc *desc); - unsigned int processor_id; EXPORT_SYMBOL(processor_id); unsigned int __machine_arch_type __read_mostly; @@ -887,10 +880,7 @@ static void __init request_standard_resources(const struct machine_desc *mdesc) */ boot_alias_start = phys_to_idmap(start); if (arm_has_idmap_alias() && boot_alias_start != IDMAP_INVALID_ADDR) { - res = memblock_alloc(sizeof(*res), SMP_CACHE_BYTES); - if (!res) - panic("%s: Failed to allocate %zu bytes\n", - __func__, sizeof(*res)); + res = memblock_alloc_or_panic(sizeof(*res), SMP_CACHE_BYTES); res->name = "System RAM (boot alias)"; res->start = boot_alias_start; res->end = phys_to_idmap(res_end); @@ -898,10 +888,7 @@ static void __init request_standard_resources(const struct machine_desc *mdesc) request_resource(&iomem_resource, res); } - res = memblock_alloc(sizeof(*res), SMP_CACHE_BYTES); - if (!res) - panic("%s: Failed to allocate %zu bytes\n", __func__, - sizeof(*res)); + res = memblock_alloc_or_panic(sizeof(*res), SMP_CACHE_BYTES); res->name = "System RAM"; res->start = start; res->end = res_end; @@ -935,9 +922,8 @@ static void __init request_standard_resources(const struct machine_desc *mdesc) request_resource(&ioport_resource, &lp2); } -#if defined(CONFIG_VGA_CONSOLE) || defined(CONFIG_DUMMY_CONSOLE) || \ - defined(CONFIG_EFI) -struct screen_info screen_info = { +#if defined(CONFIG_VGA_CONSOLE) +struct screen_info vgacon_screen_info = { .orig_video_lines = 30, .orig_video_cols = 80, .orig_video_mode = 0, @@ -987,7 +973,7 @@ static int __init init_machine_late(void) } late_initcall(init_machine_late); -#ifdef CONFIG_KEXEC +#ifdef CONFIG_CRASH_RESERVE /* * The crash region must be aligned to 128MB to avoid * zImage relocating below the reserved region. @@ -1017,7 +1003,8 @@ static void __init reserve_crashkernel(void) total_mem = get_total_mem(); ret = parse_crashkernel(boot_command_line, total_mem, - &crash_size, &crash_base); + &crash_size, &crash_base, + NULL, NULL, NULL); /* invalid value specified or crashkernel=0 */ if (ret || !crash_size) return; @@ -1073,7 +1060,7 @@ static void __init reserve_crashkernel(void) } #else static inline void reserve_crashkernel(void) {} -#endif /* CONFIG_KEXEC */ +#endif /* CONFIG_CRASH_RESERVE*/ void __init hyp_mode_check(void) { @@ -1142,7 +1129,7 @@ void __init setup_arch(char **cmdline_p) setup_initial_init_mm(_text, _etext, _edata, _end); /* populate cmd_line too for later use, preserving boot_command_line */ - strlcpy(cmd_line, boot_command_line, COMMAND_LINE_SIZE); + strscpy(cmd_line, boot_command_line, COMMAND_LINE_SIZE); *cmdline_p = cmd_line; early_fixmap_init(); @@ -1198,13 +1185,9 @@ void __init setup_arch(char **cmdline_p) reserve_crashkernel(); -#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER - handle_arch_irq = mdesc->handle_irq; -#endif - #ifdef CONFIG_VT #if defined(CONFIG_VGA_CONSOLE) - conswitchp = &vga_con; + vgacon_register_screen(&vgacon_screen_info); #endif #endif @@ -1212,20 +1195,10 @@ void __init setup_arch(char **cmdline_p) mdesc->init_early(); } - -static int __init topology_init(void) +bool arch_cpu_is_hotpluggable(int num) { - int cpu; - - for_each_possible_cpu(cpu) { - struct cpuinfo_arm *cpuinfo = &per_cpu(cpu_data, cpu); - cpuinfo->cpu.hotpluggable = platform_can_hotplug_cpu(cpu); - register_cpu(&cpuinfo->cpu, cpu); - } - - return 0; + return platform_can_hotplug_cpu(num); } -subsys_initcall(topology_init); #ifdef CONFIG_HAVE_PROC_CPU static int __init proc_cpu_init(void) diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c index e07f359254c3..79a6730fa0eb 100644 --- a/arch/arm/kernel/signal.c +++ b/arch/arm/kernel/signal.c @@ -18,6 +18,7 @@ #include <asm/traps.h> #include <asm/unistd.h> #include <asm/vfp.h> +#include <asm/syscalls.h> #include "signal.h" @@ -681,7 +682,7 @@ asmlinkage void do_rseq_syscall(struct pt_regs *regs) */ static_assert(NSIGILL == 11); static_assert(NSIGFPE == 15); -static_assert(NSIGSEGV == 9); +static_assert(NSIGSEGV == 10); static_assert(NSIGBUS == 5); static_assert(NSIGTRAP == 6); static_assert(NSIGCHLD == 6); diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S index a86a1d4f3461..93afd1005b43 100644 --- a/arch/arm/kernel/sleep.S +++ b/arch/arm/kernel/sleep.S @@ -127,6 +127,10 @@ cpu_resume_after_mmu: instr_sync #endif bl cpu_init @ restore the und/abt/irq banked regs +#if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK) + mov r0, sp + bl kasan_unpoison_task_stack_below +#endif mov r0, #0 @ return zero on success ldmfd sp!, {r4 - r11, pc} ENDPROC(cpu_resume_after_mmu) diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 36e6efad89f3..50999886a8b5 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -48,7 +48,6 @@ #include <asm/mach/arch.h> #include <asm/mpu.h> -#define CREATE_TRACE_POINTS #include <trace/events/ipi.h> /* @@ -289,15 +288,11 @@ int __cpu_disable(void) } /* - * called on the thread which is asking for a CPU to be shutdown - - * waits until shutdown has completed, or it is timed out. + * called on the thread which is asking for a CPU to be shutdown after the + * shutdown completed. */ -void __cpu_die(unsigned int cpu) +void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) { - if (!cpu_wait_death(cpu, 5)) { - pr_err("CPU%u: cpu didn't die\n", cpu); - return; - } pr_debug("CPU%u: shutdown\n", cpu); clear_tasks_mm_cpumask(cpu); @@ -320,7 +315,7 @@ void __cpu_die(unsigned int cpu) * of the other hotplug-cpu capable cores, so presumably coming * out of idle fixes this. */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { unsigned int cpu = smp_processor_id(); @@ -337,11 +332,11 @@ void arch_cpu_idle_dead(void) flush_cache_louis(); /* - * Tell __cpu_die() that this CPU is now safe to dispose of. Once - * this returns, power and/or clocks can be removed at any point - * from this CPU and its cache by platform_cpu_kill(). + * Tell cpuhp_bp_sync_dead() that this CPU is now safe to dispose + * of. Once this returns, power and/or clocks can be removed at + * any point from this CPU and its cache by platform_cpu_kill(). */ - (void)cpu_report_death(); + cpuhp_ap_report_dead(); /* * Ensure that the cache lines associated with that completion are @@ -382,6 +377,8 @@ void arch_cpu_idle_dead(void) : "r" (task_stack_page(current) + THREAD_SIZE - 8), "r" (current) : "r0"); + + unreachable(); } #endif /* CONFIG_HOTPLUG_CPU */ @@ -554,7 +551,8 @@ void show_ipi_list(struct seq_file *p, int prec) if (!ipi_desc[i]) continue; - seq_printf(p, "%*s%u: ", prec - 1, "IPI", i); + seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, + prec >= 4 ? " " : ""); for_each_online_cpu(cpu) seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu)); @@ -638,7 +636,7 @@ static void do_handle_IPI(int ipinr) unsigned int cpu = smp_processor_id(); if ((unsigned)ipinr < NR_IPI) - trace_ipi_entry_rcuidle(ipi_types[ipinr]); + trace_ipi_entry(ipi_types[ipinr]); switch (ipinr) { case IPI_WAKEUP: @@ -685,7 +683,7 @@ static void do_handle_IPI(int ipinr) } if ((unsigned)ipinr < NR_IPI) - trace_ipi_exit_rcuidle(ipi_types[ipinr]); + trace_ipi_exit(ipi_types[ipinr]); } /* Legacy version, should go away once all irqchips have been converted */ @@ -708,7 +706,7 @@ static irqreturn_t ipi_handler(int irq, void *data) static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) { - trace_ipi_raise_rcuidle(target, ipi_types[ipinr]); + trace_ipi_raise(target, ipi_types[ipinr]); __ipi_send_mask(ipi_desc[ipinr], target); } @@ -747,7 +745,7 @@ void __init set_smp_ipi_range(int ipi_base, int n) ipi_setup(smp_processor_id()); } -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); } @@ -777,7 +775,7 @@ void smp_send_stop(void) * kdump fails. So split out the panic_smp_self_stop() and add * set_cpu_online(smp_processor_id(), false). */ -void panic_smp_self_stop(void) +void __noreturn panic_smp_self_stop(void) { pr_debug("CPU %u will stop doing anything useful since another CPU has paniced\n", smp_processor_id()); @@ -849,7 +847,7 @@ static void raise_nmi(cpumask_t *mask) __ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask); } -void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) +void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) { - nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_nmi); + nmi_trigger_cpumask_backtrace(mask, exclude_cpu, raise_nmi); } diff --git a/arch/arm/kernel/smp_twd.c b/arch/arm/kernel/smp_twd.c index 9a14f721a2b0..42a3706e16a6 100644 --- a/arch/arm/kernel/smp_twd.c +++ b/arch/arm/kernel/smp_twd.c @@ -93,7 +93,6 @@ static void twd_timer_stop(void) { struct clock_event_device *clk = raw_cpu_ptr(twd_evt); - twd_shutdown(clk); disable_percpu_irq(clk->irq); } diff --git a/arch/arm/kernel/suspend.c b/arch/arm/kernel/suspend.c index 43f0a3ebf390..58a6441b58c4 100644 --- a/arch/arm/kernel/suspend.c +++ b/arch/arm/kernel/suspend.c @@ -8,10 +8,11 @@ #include <asm/bugs.h> #include <asm/cacheflush.h> #include <asm/idmap.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/smp_plat.h> #include <asm/suspend.h> #include <asm/tlbflush.h> +#include <asm/uaccess.h> extern int __cpu_suspend(unsigned long, int (*)(unsigned long), u32 cpuid); extern void cpu_resume_mmu(void); @@ -27,6 +28,13 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) return -EINVAL; /* + * Needed for the MMU disabling/enabing code to be able to run from + * TTBR0 addresses. + */ + if (IS_ENABLED(CONFIG_CPU_TTBR0_PAN)) + uaccess_save_and_enable(); + + /* * Function graph tracer state gets incosistent when the kernel * calls functions that never return (aka suspend finishers) hence * disable graph tracing during their execution. diff --git a/arch/arm/kernel/sys_arm.c b/arch/arm/kernel/sys_arm.c index a5f183cfecb1..0141e9bb02e8 100644 --- a/arch/arm/kernel/sys_arm.c +++ b/arch/arm/kernel/sys_arm.c @@ -24,6 +24,7 @@ #include <linux/ipc.h> #include <linux/uaccess.h> #include <linux/slab.h> +#include <asm/syscalls.h> /* * Since loff_t is a 64 bit type we avoid a lot of ABI hassle diff --git a/arch/arm/kernel/sys_oabi-compat.c b/arch/arm/kernel/sys_oabi-compat.c index 68112c172025..2944721e82a2 100644 --- a/arch/arm/kernel/sys_oabi-compat.c +++ b/arch/arm/kernel/sys_oabi-compat.c @@ -10,6 +10,8 @@ * Copyright: MontaVista Software, Inc. */ +#include <asm/syscalls.h> + /* * The legacy ABI and the new ARM EABI have different rules making some * syscalls incompatible especially with structure arguments. @@ -73,6 +75,7 @@ #include <linux/syscalls.h> #include <linux/errno.h> #include <linux/fs.h> +#include <linux/filelock.h> #include <linux/cred.h> #include <linux/fcntl.h> #include <linux/eventpoll.h> @@ -232,23 +235,23 @@ asmlinkage long sys_oabi_fcntl64(unsigned int fd, unsigned int cmd, unsigned long arg) { void __user *argp = (void __user *)arg; - struct fd f = fdget_raw(fd); + CLASS(fd_raw, f)(fd); struct flock64 flock; - long err = -EBADF; + long err; - if (!f.file) - goto out; + if (fd_empty(f)) + return -EBADF; switch (cmd) { case F_GETLK64: case F_OFD_GETLK: - err = security_file_fcntl(f.file, cmd, arg); + err = security_file_fcntl(fd_file(f), cmd, arg); if (err) break; err = get_oabi_flock(&flock, argp); if (err) break; - err = fcntl_getlk64(f.file, cmd, &flock); + err = fcntl_getlk64(fd_file(f), cmd, &flock); if (!err) err = put_oabi_flock(&flock, argp); break; @@ -256,20 +259,18 @@ asmlinkage long sys_oabi_fcntl64(unsigned int fd, unsigned int cmd, case F_SETLKW64: case F_OFD_SETLK: case F_OFD_SETLKW: - err = security_file_fcntl(f.file, cmd, arg); + err = security_file_fcntl(fd_file(f), cmd, arg); if (err) break; err = get_oabi_flock(&flock, argp); if (err) break; - err = fcntl_setlk64(fd, f.file, cmd, &flock); + err = fcntl_setlk64(fd, fd_file(f), cmd, &flock); break; default: err = sys_fcntl64(fd, cmd, arg); break; } - fdput(f); -out: return err; } diff --git a/arch/arm/kernel/tcm.c b/arch/arm/kernel/tcm.c index d3a85f01b328..f59927bcfbce 100644 --- a/arch/arm/kernel/tcm.c +++ b/arch/arm/kernel/tcm.c @@ -15,7 +15,7 @@ #include <linux/string.h> /* memcpy */ #include <asm/cputype.h> #include <asm/mach/map.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/system_info.h> #include <asm/traps.h> #include <asm/tcm.h> diff --git a/arch/arm/kernel/topology.c b/arch/arm/kernel/topology.c index ef0058de432b..2336ee2aa44a 100644 --- a/arch/arm/kernel/topology.c +++ b/arch/arm/kernel/topology.c @@ -42,7 +42,7 @@ * can take this difference into account during load balance. A per cpu * structure is preferred because each CPU updates its own cpu_capacity field * during the load balance except for idle cores. One idle core is selected - * to run the rebalance_domains for all idle cores and the cpu_capacity can be + * to run the sched_balance_domains for all idle cores and the cpu_capacity can be * updated during this sequence. */ diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 40c7c807d67f..afbd2ebe5c39 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -26,6 +26,7 @@ #include <linux/sched/debug.h> #include <linux/sched/task_stack.h> #include <linux/irq.h> +#include <linux/vmalloc.h> #include <linux/atomic.h> #include <asm/cacheflush.h> @@ -220,7 +221,7 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, unsigned int fp, mode; int ok = 1; - printk("%sBacktrace: ", loglvl); + printk("%sCall trace: ", loglvl); if (!tsk) tsk = current; @@ -257,13 +258,6 @@ void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl) barrier(); } -#ifdef CONFIG_PREEMPT -#define S_PREEMPT " PREEMPT" -#elif defined(CONFIG_PREEMPT_RT) -#define S_PREEMPT " PREEMPT_RT" -#else -#define S_PREEMPT "" -#endif #ifdef CONFIG_SMP #define S_SMP " SMP" #else @@ -281,8 +275,8 @@ static int __die(const char *str, int err, struct pt_regs *regs) static int die_counter; int ret; - pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP S_ISA "\n", - str, err, ++die_counter); + pr_emerg("Internal error: %s: %x [#%d]" S_SMP S_ISA "\n", + str, err, ++die_counter); /* trap and error numbers are mostly meaningless on ARM */ ret = notify_die(DIE_OOPS, str, regs, err, tsk->thread.trap_no, SIGSEGV); @@ -569,6 +563,7 @@ static int bad_syscall(int n, struct pt_regs *regs) static inline int __do_cache_op(unsigned long start, unsigned long end) { + unsigned int ua_flags; int ret; do { @@ -577,7 +572,9 @@ __do_cache_op(unsigned long start, unsigned long end) if (fatal_signal_pending(current)) return 0; + ua_flags = uaccess_save_and_enable(); ret = flush_icache_user_range(start, start + chunk); + uaccess_restore(ua_flags); if (ret) return ret; @@ -756,6 +753,7 @@ void __readwrite_bug(const char *fn) } EXPORT_SYMBOL(__readwrite_bug); +#ifdef CONFIG_MMU void __pte_error(const char *file, int line, pte_t pte) { pr_err("%s:%d: bad pte %08llx.\n", file, line, (long long)pte_val(pte)); @@ -770,6 +768,7 @@ void __pgd_error(const char *file, int line, pgd_t pgd) { pr_err("%s:%d: bad pgd %08llx.\n", file, line, (long long)pgd_val(pgd)); } +#endif asmlinkage void __div0(void) { diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index 53be7ea6181b..f60547dadc93 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -308,6 +308,29 @@ static int unwind_exec_pop_subset_r0_to_r3(struct unwind_ctrl_block *ctrl, return URC_OK; } +static unsigned long unwind_decode_uleb128(struct unwind_ctrl_block *ctrl) +{ + unsigned long bytes = 0; + unsigned long insn; + unsigned long result = 0; + + /* + * unwind_get_byte() will advance `ctrl` one instruction at a time, so + * loop until we get an instruction byte where bit 7 is not set. + * + * Note: This decodes a maximum of 4 bytes to output 28 bits data where + * max is 0xfffffff: that will cover a vsp increment of 1073742336, hence + * it is sufficient for unwinding the stack. + */ + do { + insn = unwind_get_byte(ctrl); + result |= (insn & 0x7f) << (bytes * 7); + bytes++; + } while (!!(insn & 0x80) && (bytes != sizeof(result))); + + return result; +} + /* * Execute the current unwind instruction. */ @@ -361,7 +384,7 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) if (ret) goto error; } else if (insn == 0xb2) { - unsigned long uleb128 = unwind_get_byte(ctrl); + unsigned long uleb128 = unwind_decode_uleb128(ctrl); ctrl->vrs[SP] += 0x204 + (uleb128 << 2); } else { @@ -501,6 +524,8 @@ void unwind_backtrace(struct pt_regs *regs, struct task_struct *tsk, { struct stackframe frame; + printk("%sCall trace: ", loglvl); + pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk); if (!tsk) diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c index 3408269d19c7..e38a30477f3d 100644 --- a/arch/arm/kernel/vdso.c +++ b/arch/arm/kernel/vdso.c @@ -7,6 +7,7 @@ */ #include <linux/cache.h> +#include <linux/vdso_datastore.h> #include <linux/elf.h> #include <linux/err.h> #include <linux/kernel.h> @@ -14,14 +15,12 @@ #include <linux/of.h> #include <linux/printk.h> #include <linux/slab.h> -#include <linux/timekeeper_internal.h> #include <linux/vmalloc.h> #include <asm/arch_timer.h> #include <asm/barrier.h> #include <asm/cacheflush.h> #include <asm/page.h> #include <asm/vdso.h> -#include <asm/vdso_datapage.h> #include <clocksource/arm_arch_timer.h> #include <vdso/helpers.h> #include <vdso/vsyscall.h> @@ -35,18 +34,6 @@ extern char vdso_start[], vdso_end[]; /* Total number of pages needed for the data and text portions of the VDSO. */ unsigned int vdso_total_pages __ro_after_init; -/* - * The VDSO data page. - */ -static union vdso_data_store vdso_data_store __page_aligned_data; -struct vdso_data *vdso_data = vdso_data_store.data; - -static struct page *vdso_data_page __ro_after_init; -static const struct vm_special_mapping vdso_data_mapping = { - .name = "[vvar]", - .pages = &vdso_data_page, -}; - static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma) { @@ -67,11 +54,9 @@ struct elfinfo { char *dynstr; /* ptr to .dynstr section */ }; -/* Cached result of boot-time check for whether the arch timer exists, - * and if so, whether the virtual counter is useable. +/* Boot-time check for whether the arch timer exists, and if so, + * whether the virtual counter is usable. */ -bool cntvct_ok __ro_after_init; - static bool __init cntvct_functional(void) { struct device_node *np; @@ -135,7 +120,7 @@ static Elf32_Sym * __init find_symbol(struct elfinfo *lib, const char *symname) if (lib->dynsym[i].st_name == 0) continue; - strlcpy(name, lib->dynstr + lib->dynsym[i].st_name, + strscpy(name, lib->dynstr + lib->dynsym[i].st_name, MAX_SYMNAME); c = strchr(name, '@'); if (c) @@ -172,7 +157,7 @@ static void __init patch_vdso(void *ehdr) * want programs to incur the slight additional overhead of * dispatching through the VDSO only to fall back to syscalls. */ - if (!cntvct_ok) { + if (!cntvct_functional()) { vdso_nullpatch_one(&einfo, "__vdso_gettimeofday"); vdso_nullpatch_one(&einfo, "__vdso_clock_gettime"); vdso_nullpatch_one(&einfo, "__vdso_clock_gettime64"); @@ -197,9 +182,6 @@ static int __init vdso_init(void) if (vdso_text_pagelist == NULL) return -ENOMEM; - /* Grab the VDSO data page. */ - vdso_data_page = virt_to_page(vdso_data); - /* Grab the VDSO text pages. */ for (i = 0; i < text_pages; i++) { struct page *page; @@ -210,27 +192,16 @@ static int __init vdso_init(void) vdso_text_mapping.pages = vdso_text_pagelist; - vdso_total_pages = 1; /* for the data/vvar page */ + vdso_total_pages = VDSO_NR_PAGES; /* for the data/vvar pages */ vdso_total_pages += text_pages; - cntvct_ok = cntvct_functional(); - patch_vdso(vdso_start); return 0; } arch_initcall(vdso_init); -static int install_vvar(struct mm_struct *mm, unsigned long addr) -{ - struct vm_area_struct *vma; - - vma = _install_special_mapping(mm, addr, PAGE_SIZE, - VM_READ | VM_MAYREAD, - &vdso_data_mapping); - - return PTR_ERR_OR_ZERO(vma); -} +static_assert(__VDSO_PAGES == VDSO_NR_PAGES); /* assumes mmap_lock is write-locked */ void arm_install_vdso(struct mm_struct *mm, unsigned long addr) @@ -243,12 +214,12 @@ void arm_install_vdso(struct mm_struct *mm, unsigned long addr) if (vdso_text_pagelist == NULL) return; - if (install_vvar(mm, addr)) + if (IS_ERR(vdso_install_vvar_mapping(mm, addr))) return; - /* Account for vvar page. */ - addr += PAGE_SIZE; - len = (vdso_total_pages - 1) << PAGE_SHIFT; + /* Account for vvar pages. */ + addr += VDSO_NR_PAGES * PAGE_SIZE; + len = (vdso_total_pages - VDSO_NR_PAGES) << PAGE_SHIFT; vma = _install_special_mapping(mm, addr, len, VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, diff --git a/arch/arm/kernel/vmcore_info.c b/arch/arm/kernel/vmcore_info.c new file mode 100644 index 000000000000..1437aba47787 --- /dev/null +++ b/arch/arm/kernel/vmcore_info.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/vmcore_info.h> + +void arch_crash_save_vmcoreinfo(void) +{ +#ifdef CONFIG_ARM_LPAE + VMCOREINFO_CONFIG(ARM_LPAE); +#endif +} diff --git a/arch/arm/kernel/vmlinux-xip.lds.S b/arch/arm/kernel/vmlinux-xip.lds.S index 76678732c60d..f2e8d4fac068 100644 --- a/arch/arm/kernel/vmlinux-xip.lds.S +++ b/arch/arm/kernel/vmlinux-xip.lds.S @@ -12,9 +12,8 @@ #include <asm/vmlinux.lds.h> #include <asm/cache.h> #include <asm/thread_info.h> -#include <asm/memory.h> -#include <asm/mpu.h> #include <asm/page.h> +#include <asm/mpu.h> OUTPUT_ARCH(arm) ENTRY(stext) @@ -84,7 +83,7 @@ SECTIONS } .init.arch.info : { __arch_info_begin = .; - *(.arch.info.init) + KEEP(*(.arch.info.init)) __arch_info_end = .; } .init.tagtable : { diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index aa12b65a7fd6..d592a203f9c6 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -12,9 +12,8 @@ #include <asm/vmlinux.lds.h> #include <asm/cache.h> #include <asm/thread_info.h> -#include <asm/memory.h> -#include <asm/mpu.h> #include <asm/page.h> +#include <asm/mpu.h> OUTPUT_ARCH(arm) ENTRY(stext) @@ -100,7 +99,7 @@ SECTIONS } .init.arch.info : { __arch_info_begin = .; - *(.arch.info.init) + KEEP(*(.arch.info.init)) __arch_info_end = .; } .init.tagtable : { @@ -117,7 +116,7 @@ SECTIONS #endif .init.pv_table : { __pv_table_begin = .; - *(.pv_table) + KEEP(*(.pv_table)) __pv_table_end = .; } diff --git a/arch/arm/kernel/xscale-cp0.c b/arch/arm/kernel/xscale-cp0.c index ed4f6e77616d..00d00d3aae97 100644 --- a/arch/arm/kernel/xscale-cp0.c +++ b/arch/arm/kernel/xscale-cp0.c @@ -166,6 +166,7 @@ static int __init xscale_cp0_init(void) pr_info("XScale iWMMXt coprocessor detected.\n"); elf_hwcap |= HWCAP_IWMMXT; thread_register_notifier(&iwmmxt_notifier_block); + register_iwmmxt_undef_handler(); #endif } else { pr_info("XScale DSP coprocessor detected.\n"); |