diff options
Diffstat (limited to 'arch/arm/mm')
87 files changed, 4114 insertions, 3152 deletions
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 65e4482e3849..5c1023a6d78c 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -386,6 +386,7 @@ config CPU_V6 select CPU_PABRT_V6 select CPU_THUMB_CAPABLE select CPU_TLB_V6 if MMU + select SMP_ON_UP if SMP # ARMv6k config CPU_V6K @@ -402,7 +403,7 @@ config CPU_V6K select CPU_THUMB_CAPABLE select CPU_TLB_V6 if MMU -# ARMv7 +# ARMv7 and ARMv8 architectures config CPU_V7 bool select CPU_32v6K @@ -601,8 +602,6 @@ config CPU_TLB_V6 config CPU_TLB_V7 bool -config VERIFY_PERMISSION_FAULT - bool endif config CPU_HAS_ASID @@ -632,13 +631,16 @@ config CPU_USE_DOMAINS bool help This option enables or disables the use of domain switching - via the set_fs() function. + using the DACR (domain access control register) to protect memory + domains from each other. In Linux we use three domains: kernel, user + and IO. The domains are used to protect userspace from kernelspace + and to handle IO-space as a special type of memory by assigning + manager or client roles to running code (such as a process). config CPU_V7M_NUM_IRQ int "Number of external interrupts connected to the NVIC" depends on CPU_V7M default 90 if ARCH_STM32 - default 38 if ARCH_EFM32 default 112 if SOC_VF610 default 240 help @@ -678,7 +680,7 @@ config ARM_PV_FIXUP config ARM_THUMB bool "Support Thumb user binaries" if !CPU_THUMBONLY && EXPERT - depends on CPU_THUMB_CAPABLE + depends on CPU_THUMB_CAPABLE && !CPU_32v4 default y help Say Y if you want to include kernel support for running user space @@ -710,7 +712,7 @@ config ARM_VIRT_EXT assistance. A compliant bootloader is required in order to make maximum - use of this feature. Refer to Documentation/arm/booting.rst for + use of this feature. Refer to Documentation/arch/arm/booting.rst for details. config SWP_EMULATE @@ -740,19 +742,34 @@ config SWP_EMULATE If unsure, say Y. +choice + prompt "CPU Endianness" + default CPU_LITTLE_ENDIAN + +config CPU_LITTLE_ENDIAN + bool "Built little-endian kernel" + help + Say Y if you plan on running a kernel in little-endian mode. + This is the default and is used in practically all modern user + space builds. + config CPU_BIG_ENDIAN bool "Build big-endian kernel" - depends on ARCH_SUPPORTS_BIG_ENDIAN + depends on !LD_IS_LLD help Say Y if you plan on running a kernel in big-endian mode. - Note that your board must be properly built and your board - port must properly enable any big-endian related features - of your chipset/board/processor. + This works on many machines using ARMv6 or newer processors + but requires big-endian user space. + + The only ARMv5 platform with big-endian support is + Intel IXP4xx. + +endchoice config CPU_ENDIAN_BE8 bool depends on CPU_BIG_ENDIAN - default CPU_V6 || CPU_V6K || CPU_V7 + default CPU_V6 || CPU_V6K || CPU_V7 || CPU_V7M help Support for the BE-8 (big-endian) mode on ARMv6 and ARMv7 processors. @@ -832,6 +849,7 @@ config CPU_BPREDICT_DISABLE config CPU_SPECTRE bool + select GENERIC_CPU_VULNERABILITIES config HARDEN_BRANCH_PREDICTOR bool "Harden the branch predictor against aliasing attacks" if EXPERT @@ -852,6 +870,16 @@ config HARDEN_BRANCH_PREDICTOR If unsure, say Y. +config HARDEN_BRANCH_HISTORY + bool "Harden Spectre style attacks against branch history" if EXPERT + depends on CPU_SPECTRE + default y + help + Speculation attacks against some high-performance processors can + make use of branch history to influence future speculation. When + taking an exception, a sequence of branches overwrites the branch + history, or branch history is invalidated. + config TLS_REG_EMUL bool select NEED_KUSER_HELPERS @@ -876,7 +904,7 @@ config KUSER_HELPERS the CPU type fitted to the system. This permits binaries to be run on ARMv4 through to ARMv7 without modification. - See Documentation/arm/kernel_user_helpers.rst for details. + See Documentation/arch/arm/kernel_user_helpers.rst for details. However, the fixed address nature of these helpers can be used by ROP (return orientated programming) authors when creating @@ -900,6 +928,7 @@ config VDSO select GENERIC_TIME_VSYSCALL select GENERIC_VDSO_32 select GENERIC_GETTIMEOFDAY + select GENERIC_VDSO_DATA_STORE help Place in the process address space an ELF shared object providing fast implementations of gettimeofday and @@ -909,24 +938,6 @@ config VDSO You must have glibc 2.22 or later for programs to seamlessly take advantage of this. -config DMA_CACHE_RWFO - bool "Enable read/write for ownership DMA cache maintenance" - depends on CPU_V6K && SMP - default y - help - The Snoop Control Unit on ARM11MPCore does not detect the - cache maintenance operations and the dma_{map,unmap}_area() - functions may leave stale cache entries on other CPUs. By - enabling this option, Read or Write For Ownership in the ARMv6 - DMA cache maintenance functions is performed. These LDR/STR - instructions change the cache line state to shared or modified - so that the cache operation has the desired effect. - - Note that the workaround is only valid on processors that do - not perform speculative loads into the D-cache. For such - processors, if cache maintenance operations are not broadcast - in hardware, other workarounds are needed (e.g. cache - maintenance broadcasting in software via FIQ). config OUTER_CACHE bool @@ -1112,12 +1123,6 @@ config ARM_DMA_MEM_BUFFERABLE config ARM_HEAVY_MB bool -config ARCH_SUPPORTS_BIG_ENDIAN - bool - help - This option specifies the architecture can support big endian - operation. - config DEBUG_ALIGN_RODATA bool "Make rodata strictly non-executable" depends on STRICT_KERNEL_RWX diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile index 7cb1699fbfc4..a195cd1d3e6d 100644 --- a/arch/arm/mm/Makefile +++ b/arch/arm/mm/Makefile @@ -7,6 +7,7 @@ obj-y := extable.o fault.o init.o iomap.o obj-y += dma-mapping$(MMUEXT).o obj-$(CONFIG_MMU) += fault-armv.o flush.o idmap.o ioremap.o \ mmap.o pgd.o mmu.o pageattr.o +KASAN_SANITIZE_mmu.o := n ifneq ($(CONFIG_MMU),y) obj-y += nommu.o @@ -16,11 +17,10 @@ endif obj-$(CONFIG_ARM_PTDUMP_CORE) += dump.o obj-$(CONFIG_ARM_PTDUMP_DEBUGFS) += ptdump_debugfs.o obj-$(CONFIG_MODULES) += proc-syms.o +KASAN_SANITIZE_physaddr.o := n obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ALIGNMENT_TRAP) += alignment.o -obj-$(CONFIG_HIGHMEM) += highmem.o -obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_ARM_PV_FIXUP) += pv-fixup-asm.o obj-$(CONFIG_CPU_ABRT_NOMMU) += abort-nommu.o @@ -32,9 +32,6 @@ obj-$(CONFIG_CPU_ABRT_EV5TJ) += abort-ev5tj.o obj-$(CONFIG_CPU_ABRT_EV6) += abort-ev6.o obj-$(CONFIG_CPU_ABRT_EV7) += abort-ev7.o -AFLAGS_abort-ev6.o :=-Wa,-march=armv6k -AFLAGS_abort-ev7.o :=-Wa,-march=armv7-a - obj-$(CONFIG_CPU_PABRT_LEGACY) += pabort-legacy.o obj-$(CONFIG_CPU_PABRT_V6) += pabort-v6.o obj-$(CONFIG_CPU_PABRT_V7) += pabort-v7.o @@ -47,10 +44,7 @@ obj-$(CONFIG_CPU_CACHE_V7) += cache-v7.o obj-$(CONFIG_CPU_CACHE_FA) += cache-fa.o obj-$(CONFIG_CPU_CACHE_NOP) += cache-nop.o obj-$(CONFIG_CPU_CACHE_V7M) += cache-v7m.o - -AFLAGS_cache-v6.o :=-Wa,-march=armv6 -AFLAGS_cache-v7.o :=-Wa,-march=armv7-a -AFLAGS_cache-v7m.o :=-Wa,-march=armv7-m +obj-y += cache.o obj-$(CONFIG_CPU_COPY_V4WT) += copypage-v4wt.o obj-$(CONFIG_CPU_COPY_V4WB) += copypage-v4wb.o @@ -61,8 +55,6 @@ obj-$(CONFIG_CPU_XSCALE) += copypage-xscale.o obj-$(CONFIG_CPU_XSC3) += copypage-xsc3.o obj-$(CONFIG_CPU_COPY_FA) += copypage-fa.o -CFLAGS_copypage-feroceon.o := -march=armv5te - obj-$(CONFIG_CPU_TLB_V4WT) += tlb-v4.o obj-$(CONFIG_CPU_TLB_V4WB) += tlb-v4wb.o obj-$(CONFIG_CPU_TLB_V4WBI) += tlb-v4wbi.o @@ -70,9 +62,7 @@ obj-$(CONFIG_CPU_TLB_FEROCEON) += tlb-v4wbi.o # reuse v4wbi TLB functions obj-$(CONFIG_CPU_TLB_V6) += tlb-v6.o obj-$(CONFIG_CPU_TLB_V7) += tlb-v7.o obj-$(CONFIG_CPU_TLB_FA) += tlb-fa.o - -AFLAGS_tlb-v6.o :=-Wa,-march=armv6 -AFLAGS_tlb-v7.o :=-Wa,-march=armv7-a +obj-y += tlb.o obj-$(CONFIG_CPU_ARM7TDMI) += proc-arm7tdmi.o obj-$(CONFIG_CPU_ARM720T) += proc-arm720.o @@ -99,9 +89,7 @@ obj-$(CONFIG_CPU_V6) += proc-v6.o obj-$(CONFIG_CPU_V6K) += proc-v6.o obj-$(CONFIG_CPU_V7) += proc-v7.o proc-v7-bugs.o obj-$(CONFIG_CPU_V7M) += proc-v7m.o - -AFLAGS_proc-v6.o :=-Wa,-march=armv6 -AFLAGS_proc-v7.o :=-Wa,-march=armv7-a +obj-$(CONFIG_CFI_CLANG) += proc.o obj-$(CONFIG_OUTER_CACHE) += l2c-common.o obj-$(CONFIG_CACHE_B15_RAC) += cache-b15-rac.o @@ -111,3 +99,6 @@ obj-$(CONFIG_CACHE_L2X0_PMU) += cache-l2x0-pmu.o obj-$(CONFIG_CACHE_XSC3L2) += cache-xsc3l2.o obj-$(CONFIG_CACHE_TAUROS2) += cache-tauros2.o obj-$(CONFIG_CACHE_UNIPHIER) += cache-uniphier.o + +KASAN_SANITIZE_kasan_init.o := n +obj-$(CONFIG_KASAN) += kasan_init.o diff --git a/arch/arm/mm/abort-ev6.S b/arch/arm/mm/abort-ev6.S index c58bf8b43fea..836dc1299243 100644 --- a/arch/arm/mm/abort-ev6.S +++ b/arch/arm/mm/abort-ev6.S @@ -16,6 +16,7 @@ * abort here if the I-TLB and D-TLB aren't seeing the same * picture. Unfortunately, this does happen. We live with it. */ + .arch armv6k .align 5 ENTRY(v6_early_abort) mrc p15, 0, r1, c5, c0, 0 @ get FSR diff --git a/arch/arm/mm/abort-ev7.S b/arch/arm/mm/abort-ev7.S index f7cc5d68444b..53fb41c24774 100644 --- a/arch/arm/mm/abort-ev7.S +++ b/arch/arm/mm/abort-ev7.S @@ -12,36 +12,11 @@ * * Purpose : obtain information about current aborted instruction. */ + .arch armv7-a .align 5 ENTRY(v7_early_abort) mrc p15, 0, r1, c5, c0, 0 @ get FSR mrc p15, 0, r0, c6, c0, 0 @ get FAR uaccess_disable ip @ disable userspace access - - /* - * V6 code adjusts the returned DFSR. - * New designs should not need to patch up faults. - */ - -#if defined(CONFIG_VERIFY_PERMISSION_FAULT) - /* - * Detect erroneous permission failures and fix - */ - ldr r3, =0x40d @ On permission fault - and r3, r1, r3 - cmp r3, #0x0d - bne do_DataAbort - - mcr p15, 0, r0, c7, c8, 0 @ Retranslate FAR - isb - mrc p15, 0, ip, c7, c4, 0 @ Read the PAR - and r3, ip, #0x7b @ On translation fault - cmp r3, #0x0b - bne do_DataAbort - bic r1, r1, #0xf @ Fix up FSR FS[5:0] - and ip, ip, #0x7e - orr r1, r1, ip, LSR #1 -#endif - b do_DataAbort ENDPROC(v7_early_abort) diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c index 788c5cf46de5..3c6ddb1afdc4 100644 --- a/arch/arm/mm/alignment.c +++ b/arch/arm/mm/alignment.c @@ -22,7 +22,7 @@ #include <asm/cp15.h> #include <asm/system_info.h> -#include <asm/unaligned.h> +#include <linux/unaligned.h> #include <asm/opcodes.h> #include "fault.h" @@ -162,12 +162,12 @@ static ssize_t alignment_proc_write(struct file *file, const char __user *buffer return count; } -static const struct file_operations alignment_proc_fops = { - .open = alignment_proc_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = alignment_proc_write, +static const struct proc_ops alignment_proc_ops = { + .proc_open = alignment_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = alignment_proc_write, }; #endif /* CONFIG_PROC_FS */ @@ -694,7 +694,7 @@ thumb2arm(u16 tinstr) return subset[(L<<1) | ((tinstr & (1<<8)) >> 8)] | (tinstr & 255); /* register_list */ } - /* Else, fall through - for illegal instruction case */ + fallthrough; /* for illegal instruction case */ default: return BAD_INSTR; @@ -750,7 +750,7 @@ do_alignment_t32_to_handler(u32 *pinstr, struct pt_regs *regs, case 0xe8e0: case 0xe9e0: poffset->un = (tinst2 & 0xff) << 2; - /* Fall through */ + fallthrough; case 0xe940: case 0xe9c0: @@ -774,7 +774,7 @@ static int alignment_get_arm(struct pt_regs *regs, u32 *ip, u32 *inst) if (user_mode(regs)) fault = get_user(instr, ip); else - fault = probe_kernel_address(ip, instr); + fault = get_kernel_nofault(instr, ip); *inst = __mem_to_opcode_arm(instr); @@ -789,7 +789,7 @@ static int alignment_get_thumb(struct pt_regs *regs, u16 *ip, u16 *inst) if (user_mode(regs)) fault = get_user(instr, ip); else - fault = probe_kernel_address(ip, instr); + fault = get_kernel_nofault(instr, ip); *inst = __mem_to_opcode_thumb16(instr); @@ -799,7 +799,7 @@ static int alignment_get_thumb(struct pt_regs *regs, u16 *ip, u16 *inst) static int do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs) { - union offset_union uninitialized_var(offset); + union offset_union offset; unsigned long instrptr; int (*handler)(unsigned long addr, u32 instr, struct pt_regs *regs); unsigned int type; @@ -935,6 +935,9 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs) if (type == TYPE_LDST) do_alignment_finish_ldst(addr, instr, regs, offset); + if (thumb_mode(regs)) + regs->ARM_cpsr = it_advance(regs->ARM_cpsr); + return 0; bad_or_fault: @@ -990,7 +993,7 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs) * there is no work pending for this thread. */ raw_local_irq_disable(); - if (!(current_thread_info()->flags & _TIF_WORK_MASK)) + if (!(read_thread_flags() & _TIF_WORK_MASK)) set_cr(cr_no_alignment); } @@ -1005,7 +1008,7 @@ static int __init noalign_setup(char *__unused) __setup("noalign", noalign_setup); /* - * This needs to be done after sysctl_init, otherwise sys/ will be + * This needs to be done after sysctl_init_bases(), otherwise sys/ will be * overwritten. Actually, this shouldn't be in sys/ at all since * it isn't a sysctl, and it doesn't contain sysctl information. * We now locate it in /proc/cpu/alignment instead. @@ -1016,7 +1019,7 @@ static int __init alignment_init(void) struct proc_dir_entry *res; res = proc_create("cpu/alignment", S_IWUSR | S_IRUGO, NULL, - &alignment_proc_fops); + &alignment_proc_ops); if (!res) return -ENOMEM; #endif diff --git a/arch/arm/mm/cache-b15-rac.c b/arch/arm/mm/cache-b15-rac.c index 3471fc64a3ae..6f63b90f9e1a 100644 --- a/arch/arm/mm/cache-b15-rac.c +++ b/arch/arm/mm/cache-b15-rac.c @@ -5,6 +5,7 @@ * Copyright (C) 2015-2016 Broadcom */ +#include <linux/cfi_types.h> #include <linux/err.h> #include <linux/spinlock.h> #include <linux/io.h> @@ -74,7 +75,7 @@ static inline void __b15_rac_flush(void) __raw_writel(FLUSH_RAC, b15_rac_base + rac_flush_offset); do { /* This dmb() is required to force the Bus Interface Unit - * to clean oustanding writes, and forces an idle cycle + * to clean outstanding writes, and forces an idle cycle * to be inserted. */ dmb(); @@ -358,8 +359,7 @@ static int __init b15_rac_init(void) set_bit(RAC_ENABLED, &b15_rac_flags); spin_unlock(&rac_lock); - pr_info("Broadcom Brahma-B15 readahead cache at: 0x%p\n", - b15_rac_base + RAC_CONFIG0_REG); + pr_info("%pOF: Broadcom Brahma-B15 readahead cache\n", dn); goto out; diff --git a/arch/arm/mm/cache-fa.S b/arch/arm/mm/cache-fa.S index 3a464d1649b4..4a3668b52a2d 100644 --- a/arch/arm/mm/cache-fa.S +++ b/arch/arm/mm/cache-fa.S @@ -12,8 +12,8 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> -#include <asm/memory.h> #include <asm/page.h> #include "proc-macros.S" @@ -40,11 +40,11 @@ * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(fa_flush_icache_all) +SYM_TYPED_FUNC_START(fa_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(fa_flush_icache_all) +SYM_FUNC_END(fa_flush_icache_all) /* * flush_user_cache_all() @@ -52,14 +52,14 @@ ENDPROC(fa_flush_icache_all) * Clean and invalidate all cache entries in a particular address * space. */ -ENTRY(fa_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(fa_flush_user_cache_all, fa_flush_kern_cache_all) + /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(fa_flush_kern_cache_all) +SYM_TYPED_FUNC_START(fa_flush_kern_cache_all) mov ip, #0 mov r2, #VM_EXEC __flush_whole_cache: @@ -70,6 +70,7 @@ __flush_whole_cache: mcrne p15, 0, ip, c7, c10, 4 @ drain write buffer mcrne p15, 0, ip, c7, c5, 4 @ prefetch flush ret lr +SYM_FUNC_END(fa_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -81,7 +82,7 @@ __flush_whole_cache: * - end - end address (exclusive, page aligned) * - flags - vma_area_struct flags describing address space */ -ENTRY(fa_flush_user_cache_range) +SYM_TYPED_FUNC_START(fa_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT @ total size >= limit? @@ -98,6 +99,7 @@ ENTRY(fa_flush_user_cache_range) mcrne p15, 0, ip, c7, c10, 4 @ data write barrier mcrne p15, 0, ip, c7, c5, 4 @ prefetch flush ret lr +SYM_FUNC_END(fa_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -109,8 +111,11 @@ ENTRY(fa_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(fa_coherent_kern_range) - /* fall through */ +SYM_TYPED_FUNC_START(fa_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b fa_coherent_user_range +#endif +SYM_FUNC_END(fa_coherent_kern_range) /* * coherent_user_range(start, end) @@ -122,7 +127,7 @@ ENTRY(fa_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(fa_coherent_user_range) +SYM_TYPED_FUNC_START(fa_coherent_user_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c14, 1 @ clean and invalidate D entry mcr p15, 0, r0, c7, c5, 1 @ invalidate I entry @@ -134,6 +139,7 @@ ENTRY(fa_coherent_user_range) mcr p15, 0, r0, c7, c10, 4 @ drain write buffer mcr p15, 0, r0, c7, c5, 4 @ prefetch flush ret lr +SYM_FUNC_END(fa_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -144,7 +150,7 @@ ENTRY(fa_coherent_user_range) * - addr - kernel address * - size - size of region */ -ENTRY(fa_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(fa_flush_kern_dcache_area) add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line add r0, r0, #CACHE_DLINESIZE @@ -154,6 +160,7 @@ ENTRY(fa_flush_kern_dcache_area) mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache mcr p15, 0, r0, c7, c10, 4 @ drain write buffer ret lr +SYM_FUNC_END(fa_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -204,7 +211,7 @@ fa_dma_clean_range: * - start - virtual start address of region * - end - virtual end address of region */ -ENTRY(fa_dma_flush_range) +SYM_TYPED_FUNC_START(fa_dma_flush_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D entry add r0, r0, #CACHE_DLINESIZE @@ -213,6 +220,7 @@ ENTRY(fa_dma_flush_range) mov r0, #0 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer ret lr +SYM_FUNC_END(fa_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -220,13 +228,13 @@ ENTRY(fa_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(fa_dma_map_area) +SYM_TYPED_FUNC_START(fa_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq fa_dma_clean_range bcs fa_dma_inv_range b fa_dma_flush_range -ENDPROC(fa_dma_map_area) +SYM_FUNC_END(fa_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -234,14 +242,6 @@ ENDPROC(fa_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(fa_dma_unmap_area) +SYM_TYPED_FUNC_START(fa_dma_unmap_area) ret lr -ENDPROC(fa_dma_unmap_area) - - .globl fa_flush_kern_cache_louis - .equ fa_flush_kern_cache_louis, fa_flush_kern_cache_all - - __INITDATA - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions fa +SYM_FUNC_END(fa_dma_unmap_area) diff --git a/arch/arm/mm/cache-feroceon-l2.c b/arch/arm/mm/cache-feroceon-l2.c index 5c1b7a7b9af6..25dbd84a1aaf 100644 --- a/arch/arm/mm/cache-feroceon-l2.c +++ b/arch/arm/mm/cache-feroceon-l2.c @@ -1,12 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * arch/arm/mm/cache-feroceon-l2.c - Feroceon L2 cache controller support * * Copyright (C) 2008 Marvell Semiconductor * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. - * * References: * - Unified Layer 2 Cache for Feroceon CPU Cores, * Document ID MV-S104858-00, Rev. A, October 23 2007. diff --git a/arch/arm/mm/cache-l2x0-pmu.c b/arch/arm/mm/cache-l2x0-pmu.c index 993fefdc167a..93ef0502b7ff 100644 --- a/arch/arm/mm/cache-l2x0-pmu.c +++ b/arch/arm/mm/cache-l2x0-pmu.c @@ -539,8 +539,7 @@ static __init int l2x0_pmu_init(void) * at higher frequencies. */ l2x0_pmu_poll_period = ms_to_ktime(1000); - hrtimer_init(&l2x0_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - l2x0_pmu_hrtimer.function = l2x0_pmu_poll; + hrtimer_setup(&l2x0_pmu_hrtimer, l2x0_pmu_poll, CLOCK_MONOTONIC, HRTIMER_MODE_REL); cpumask_set_cpu(0, &pmu_cpu); ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE, diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index 12c26eb88afb..43d91bfd2360 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -1249,20 +1249,28 @@ static void __init l2c310_of_parse(const struct device_node *np, ret = of_property_read_u32(np, "prefetch-data", &val); if (ret == 0) { - if (val) + if (val) { prefetch |= L310_PREFETCH_CTRL_DATA_PREFETCH; - else + *aux_val |= L310_PREFETCH_CTRL_DATA_PREFETCH; + } else { prefetch &= ~L310_PREFETCH_CTRL_DATA_PREFETCH; + *aux_val &= ~L310_PREFETCH_CTRL_DATA_PREFETCH; + } + *aux_mask &= ~L310_PREFETCH_CTRL_DATA_PREFETCH; } else if (ret != -EINVAL) { pr_err("L2C-310 OF prefetch-data property value is missing\n"); } ret = of_property_read_u32(np, "prefetch-instr", &val); if (ret == 0) { - if (val) + if (val) { prefetch |= L310_PREFETCH_CTRL_INSTR_PREFETCH; - else + *aux_val |= L310_PREFETCH_CTRL_INSTR_PREFETCH; + } else { prefetch &= ~L310_PREFETCH_CTRL_INSTR_PREFETCH; + *aux_val &= ~L310_PREFETCH_CTRL_INSTR_PREFETCH; + } + *aux_mask &= ~L310_PREFETCH_CTRL_INSTR_PREFETCH; } else if (ret != -EINVAL) { pr_err("L2C-310 OF prefetch-instr property value is missing\n"); } diff --git a/arch/arm/mm/cache-nop.S b/arch/arm/mm/cache-nop.S index 72d939ef8798..f68dde2014ee 100644 --- a/arch/arm/mm/cache-nop.S +++ b/arch/arm/mm/cache-nop.S @@ -1,47 +1,52 @@ /* SPDX-License-Identifier: GPL-2.0-only */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include "proc-macros.S" -ENTRY(nop_flush_icache_all) +/* + * These are all open-coded instead of aliased, to make clear + * what is going on here: all functions are stubbed out. + */ +SYM_TYPED_FUNC_START(nop_flush_icache_all) ret lr -ENDPROC(nop_flush_icache_all) +SYM_FUNC_END(nop_flush_icache_all) - .globl nop_flush_kern_cache_all - .equ nop_flush_kern_cache_all, nop_flush_icache_all - - .globl nop_flush_kern_cache_louis - .equ nop_flush_kern_cache_louis, nop_flush_icache_all +SYM_TYPED_FUNC_START(nop_flush_kern_cache_all) + ret lr +SYM_FUNC_END(nop_flush_kern_cache_all) - .globl nop_flush_user_cache_all - .equ nop_flush_user_cache_all, nop_flush_icache_all +SYM_TYPED_FUNC_START(nop_flush_user_cache_all) + ret lr +SYM_FUNC_END(nop_flush_user_cache_all) - .globl nop_flush_user_cache_range - .equ nop_flush_user_cache_range, nop_flush_icache_all +SYM_TYPED_FUNC_START(nop_flush_user_cache_range) + ret lr +SYM_FUNC_END(nop_flush_user_cache_range) - .globl nop_coherent_kern_range - .equ nop_coherent_kern_range, nop_flush_icache_all +SYM_TYPED_FUNC_START(nop_coherent_kern_range) + ret lr +SYM_FUNC_END(nop_coherent_kern_range) -ENTRY(nop_coherent_user_range) +SYM_TYPED_FUNC_START(nop_coherent_user_range) mov r0, 0 ret lr -ENDPROC(nop_coherent_user_range) - - .globl nop_flush_kern_dcache_area - .equ nop_flush_kern_dcache_area, nop_flush_icache_all +SYM_FUNC_END(nop_coherent_user_range) - .globl nop_dma_flush_range - .equ nop_dma_flush_range, nop_flush_icache_all - - .globl nop_dma_map_area - .equ nop_dma_map_area, nop_flush_icache_all +SYM_TYPED_FUNC_START(nop_flush_kern_dcache_area) + ret lr +SYM_FUNC_END(nop_flush_kern_dcache_area) - .globl nop_dma_unmap_area - .equ nop_dma_unmap_area, nop_flush_icache_all +SYM_TYPED_FUNC_START(nop_dma_flush_range) + ret lr +SYM_FUNC_END(nop_dma_flush_range) - __INITDATA +SYM_TYPED_FUNC_START(nop_dma_map_area) + ret lr +SYM_FUNC_END(nop_dma_map_area) - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions nop +SYM_TYPED_FUNC_START(nop_dma_unmap_area) + ret lr +SYM_FUNC_END(nop_dma_unmap_area) diff --git a/arch/arm/mm/cache-tauros2.c b/arch/arm/mm/cache-tauros2.c index 88255bea65e4..b1e1aba602f7 100644 --- a/arch/arm/mm/cache-tauros2.c +++ b/arch/arm/mm/cache-tauros2.c @@ -1,12 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * arch/arm/mm/cache-tauros2.c - Tauros2 L2 cache controller support * * Copyright (C) 2008 Marvell Semiconductor * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. - * * References: * - PJ1 CPU Core Datasheet, * Document ID MV-S104837-01, Rev 0.7, January 24 2008. diff --git a/arch/arm/mm/cache-uniphier.c b/arch/arm/mm/cache-uniphier.c index ff2881458504..84a2f17ff32d 100644 --- a/arch/arm/mm/cache-uniphier.c +++ b/arch/arm/mm/cache-uniphier.c @@ -58,11 +58,13 @@ ((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE) /** - * uniphier_cache_data - UniPhier outer cache specific data + * struct uniphier_cache_data - UniPhier outer cache specific data * * @ctrl_base: virtual base address of control registers * @rev_base: virtual base address of revision registers * @op_base: virtual base address of operation registers + * @way_ctrl_base: virtual address of the way control registers for this + * SoC revision * @way_mask: each bit specifies if the way is present * @nsets: number of associativity sets * @line_size: line size in bytes diff --git a/arch/arm/mm/cache-v4.S b/arch/arm/mm/cache-v4.S index 7787057e4990..0e94e5193dbd 100644 --- a/arch/arm/mm/cache-v4.S +++ b/arch/arm/mm/cache-v4.S @@ -6,6 +6,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/page.h> #include "proc-macros.S" @@ -15,9 +16,9 @@ * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(v4_flush_icache_all) +SYM_TYPED_FUNC_START(v4_flush_icache_all) ret lr -ENDPROC(v4_flush_icache_all) +SYM_FUNC_END(v4_flush_icache_all) /* * flush_user_cache_all() @@ -27,21 +28,22 @@ ENDPROC(v4_flush_icache_all) * * - mm - mm_struct describing address space */ -ENTRY(v4_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(v4_flush_user_cache_all, v4_flush_kern_cache_all) + /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(v4_flush_kern_cache_all) +SYM_TYPED_FUNC_START(v4_flush_kern_cache_all) #ifdef CONFIG_CPU_CP15 mov r0, #0 mcr p15, 0, r0, c7, c7, 0 @ flush ID cache ret lr #else - /* FALLTHROUGH */ + ret lr #endif +SYM_FUNC_END(v4_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -53,14 +55,15 @@ ENTRY(v4_flush_kern_cache_all) * - end - end address (exclusive, may not be aligned) * - flags - vma_area_struct flags describing address space */ -ENTRY(v4_flush_user_cache_range) +SYM_TYPED_FUNC_START(v4_flush_user_cache_range) #ifdef CONFIG_CPU_CP15 mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ flush ID cache ret lr #else - /* FALLTHROUGH */ + ret lr #endif +SYM_FUNC_END(v4_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -72,8 +75,9 @@ ENTRY(v4_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(v4_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(v4_coherent_kern_range) + ret lr +SYM_FUNC_END(v4_coherent_kern_range) /* * coherent_user_range(start, end) @@ -85,9 +89,10 @@ ENTRY(v4_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(v4_coherent_user_range) +SYM_TYPED_FUNC_START(v4_coherent_user_range) mov r0, #0 ret lr +SYM_FUNC_END(v4_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -98,8 +103,11 @@ ENTRY(v4_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(v4_flush_kern_dcache_area) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(v4_flush_kern_dcache_area) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b v4_dma_flush_range +#endif +SYM_FUNC_END(v4_flush_kern_dcache_area) /* * dma_flush_range(start, end) @@ -109,12 +117,13 @@ ENTRY(v4_flush_kern_dcache_area) * - start - virtual start address * - end - virtual end address */ -ENTRY(v4_dma_flush_range) +SYM_TYPED_FUNC_START(v4_dma_flush_range) #ifdef CONFIG_CPU_CP15 mov r0, #0 mcr p15, 0, r0, c7, c7, 0 @ flush ID cache #endif ret lr +SYM_FUNC_END(v4_dma_flush_range) /* * dma_unmap_area(start, size, dir) @@ -122,10 +131,11 @@ ENTRY(v4_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(v4_dma_unmap_area) +SYM_TYPED_FUNC_START(v4_dma_unmap_area) teq r2, #DMA_TO_DEVICE bne v4_dma_flush_range - /* FALLTHROUGH */ + ret lr +SYM_FUNC_END(v4_dma_unmap_area) /* * dma_map_area(start, size, dir) @@ -133,15 +143,6 @@ ENTRY(v4_dma_unmap_area) * - size - size of region * - dir - DMA direction */ -ENTRY(v4_dma_map_area) +SYM_TYPED_FUNC_START(v4_dma_map_area) ret lr -ENDPROC(v4_dma_unmap_area) -ENDPROC(v4_dma_map_area) - - .globl v4_flush_kern_cache_louis - .equ v4_flush_kern_cache_louis, v4_flush_kern_cache_all - - __INITDATA - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions v4 +SYM_FUNC_END(v4_dma_map_area) diff --git a/arch/arm/mm/cache-v4wb.S b/arch/arm/mm/cache-v4wb.S index 905ac2fa2b1e..ce55a2eef5da 100644 --- a/arch/arm/mm/cache-v4wb.S +++ b/arch/arm/mm/cache-v4wb.S @@ -6,8 +6,8 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> -#include <asm/memory.h> #include <asm/page.h> #include "proc-macros.S" @@ -54,11 +54,11 @@ flush_base: * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(v4wb_flush_icache_all) +SYM_TYPED_FUNC_START(v4wb_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(v4wb_flush_icache_all) +SYM_FUNC_END(v4wb_flush_icache_all) /* * flush_user_cache_all() @@ -66,14 +66,14 @@ ENDPROC(v4wb_flush_icache_all) * Clean and invalidate all cache entries in a particular address * space. */ -ENTRY(v4wb_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(v4wb_flush_user_cache_all, v4wb_flush_kern_cache_all) + /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(v4wb_flush_kern_cache_all) +SYM_TYPED_FUNC_START(v4wb_flush_kern_cache_all) mov ip, #0 mcr p15, 0, ip, c7, c5, 0 @ invalidate I cache __flush_whole_cache: @@ -94,6 +94,7 @@ __flush_whole_cache: #endif mcr p15, 0, ip, c7, c10, 4 @ drain write buffer ret lr +SYM_FUNC_END(v4wb_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -105,7 +106,7 @@ __flush_whole_cache: * - end - end address (exclusive, page aligned) * - flags - vma_area_struct flags describing address space */ -ENTRY(v4wb_flush_user_cache_range) +SYM_TYPED_FUNC_START(v4wb_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size tst r2, #VM_EXEC @ executable region? @@ -122,6 +123,7 @@ ENTRY(v4wb_flush_user_cache_range) tst r2, #VM_EXEC mcrne p15, 0, ip, c7, c10, 4 @ drain write buffer ret lr +SYM_FUNC_END(v4wb_flush_user_cache_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -132,9 +134,12 @@ ENTRY(v4wb_flush_user_cache_range) * - addr - kernel address * - size - region size */ -ENTRY(v4wb_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(v4wb_flush_kern_dcache_area) add r1, r0, r1 - /* fall through */ +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b v4wb_coherent_user_range +#endif +SYM_FUNC_END(v4wb_flush_kern_dcache_area) /* * coherent_kern_range(start, end) @@ -146,8 +151,11 @@ ENTRY(v4wb_flush_kern_dcache_area) * - start - virtual start address * - end - virtual end address */ -ENTRY(v4wb_coherent_kern_range) - /* fall through */ +SYM_TYPED_FUNC_START(v4wb_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b v4wb_coherent_user_range +#endif +SYM_FUNC_END(v4wb_coherent_kern_range) /* * coherent_user_range(start, end) @@ -159,7 +167,7 @@ ENTRY(v4wb_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(v4wb_coherent_user_range) +SYM_TYPED_FUNC_START(v4wb_coherent_user_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry mcr p15, 0, r0, c7, c6, 1 @ invalidate D entry @@ -170,7 +178,7 @@ ENTRY(v4wb_coherent_user_range) mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr - +SYM_FUNC_END(v4wb_coherent_user_range) /* * dma_inv_range(start, end) @@ -232,13 +240,13 @@ v4wb_dma_clean_range: * - size - size of region * - dir - DMA direction */ -ENTRY(v4wb_dma_map_area) +SYM_TYPED_FUNC_START(v4wb_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq v4wb_dma_clean_range bcs v4wb_dma_inv_range b v4wb_dma_flush_range -ENDPROC(v4wb_dma_map_area) +SYM_FUNC_END(v4wb_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -246,14 +254,6 @@ ENDPROC(v4wb_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(v4wb_dma_unmap_area) +SYM_TYPED_FUNC_START(v4wb_dma_unmap_area) ret lr -ENDPROC(v4wb_dma_unmap_area) - - .globl v4wb_flush_kern_cache_louis - .equ v4wb_flush_kern_cache_louis, v4wb_flush_kern_cache_all - - __INITDATA - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions v4wb +SYM_FUNC_END(v4wb_dma_unmap_area) diff --git a/arch/arm/mm/cache-v4wt.S b/arch/arm/mm/cache-v4wt.S index 0b290c25a99d..a97dc267b3b0 100644 --- a/arch/arm/mm/cache-v4wt.S +++ b/arch/arm/mm/cache-v4wt.S @@ -10,6 +10,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/page.h> #include "proc-macros.S" @@ -43,11 +44,11 @@ * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(v4wt_flush_icache_all) +SYM_TYPED_FUNC_START(v4wt_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(v4wt_flush_icache_all) +SYM_FUNC_END(v4wt_flush_icache_all) /* * flush_user_cache_all() @@ -55,14 +56,14 @@ ENDPROC(v4wt_flush_icache_all) * Invalidate all cache entries in a particular address * space. */ -ENTRY(v4wt_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(v4wt_flush_user_cache_all, v4wt_flush_kern_cache_all) + /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(v4wt_flush_kern_cache_all) +SYM_TYPED_FUNC_START(v4wt_flush_kern_cache_all) mov r2, #VM_EXEC mov ip, #0 __flush_whole_cache: @@ -70,6 +71,7 @@ __flush_whole_cache: mcrne p15, 0, ip, c7, c5, 0 @ invalidate I cache mcr p15, 0, ip, c7, c6, 0 @ invalidate D cache ret lr +SYM_FUNC_END(v4wt_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -81,7 +83,7 @@ __flush_whole_cache: * - end - end address (exclusive, page aligned) * - flags - vma_area_struct flags describing address space */ -ENTRY(v4wt_flush_user_cache_range) +SYM_TYPED_FUNC_START(v4wt_flush_user_cache_range) sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT bhs __flush_whole_cache @@ -93,6 +95,7 @@ ENTRY(v4wt_flush_user_cache_range) cmp r0, r1 blo 1b ret lr +SYM_FUNC_END(v4wt_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -104,8 +107,11 @@ ENTRY(v4wt_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(v4wt_coherent_kern_range) - /* FALLTRHOUGH */ +SYM_TYPED_FUNC_START(v4wt_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b v4wt_coherent_user_range +#endif +SYM_FUNC_END(v4wt_coherent_kern_range) /* * coherent_user_range(start, end) @@ -117,7 +123,7 @@ ENTRY(v4wt_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(v4wt_coherent_user_range) +SYM_TYPED_FUNC_START(v4wt_coherent_user_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c5, 1 @ invalidate I entry add r0, r0, #CACHE_DLINESIZE @@ -125,6 +131,7 @@ ENTRY(v4wt_coherent_user_range) blo 1b mov r0, #0 ret lr +SYM_FUNC_END(v4wt_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -135,11 +142,12 @@ ENTRY(v4wt_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(v4wt_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(v4wt_flush_kern_dcache_area) mov r2, #0 mcr p15, 0, r2, c7, c5, 0 @ invalidate I cache add r1, r0, r1 - /* fallthrough */ + b v4wt_dma_inv_range +SYM_FUNC_END(v4wt_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -167,9 +175,10 @@ v4wt_dma_inv_range: * * - start - virtual start address * - end - virtual end address - */ - .globl v4wt_dma_flush_range - .equ v4wt_dma_flush_range, v4wt_dma_inv_range +*/ +SYM_TYPED_FUNC_START(v4wt_dma_flush_range) + b v4wt_dma_inv_range +SYM_FUNC_END(v4wt_dma_flush_range) /* * dma_unmap_area(start, size, dir) @@ -177,11 +186,12 @@ v4wt_dma_inv_range: * - size - size of region * - dir - DMA direction */ -ENTRY(v4wt_dma_unmap_area) +SYM_TYPED_FUNC_START(v4wt_dma_unmap_area) add r1, r1, r0 teq r2, #DMA_TO_DEVICE bne v4wt_dma_inv_range - /* FALLTHROUGH */ + ret lr +SYM_FUNC_END(v4wt_dma_unmap_area) /* * dma_map_area(start, size, dir) @@ -189,15 +199,6 @@ ENTRY(v4wt_dma_unmap_area) * - size - size of region * - dir - DMA direction */ -ENTRY(v4wt_dma_map_area) +SYM_TYPED_FUNC_START(v4wt_dma_map_area) ret lr -ENDPROC(v4wt_dma_unmap_area) -ENDPROC(v4wt_dma_map_area) - - .globl v4wt_flush_kern_cache_louis - .equ v4wt_flush_kern_cache_louis, v4wt_flush_kern_cache_all - - __INITDATA - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions v4wt +SYM_FUNC_END(v4wt_dma_map_area) diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S index f0f65eb073e4..9f415476e218 100644 --- a/arch/arm/mm/cache-v6.S +++ b/arch/arm/mm/cache-v6.S @@ -8,6 +8,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/errno.h> #include <asm/unwind.h> @@ -19,6 +20,8 @@ #define D_CACHE_LINE_SIZE 32 #define BTB_FLUSH_SIZE 8 +.arch armv6 + /* * v6_flush_icache_all() * @@ -32,7 +35,7 @@ * r0 - set to 0 * r1 - corrupted */ -ENTRY(v6_flush_icache_all) +SYM_TYPED_FUNC_START(v6_flush_icache_all) mov r0, #0 #ifdef CONFIG_ARM_ERRATA_411920 mrs r1, cpsr @@ -49,7 +52,7 @@ ENTRY(v6_flush_icache_all) mcr p15, 0, r0, c7, c5, 0 @ invalidate I-cache #endif ret lr -ENDPROC(v6_flush_icache_all) +SYM_FUNC_END(v6_flush_icache_all) /* * v6_flush_cache_all() @@ -58,7 +61,7 @@ ENDPROC(v6_flush_icache_all) * * It is assumed that: */ -ENTRY(v6_flush_kern_cache_all) +SYM_TYPED_FUNC_START(v6_flush_kern_cache_all) mov r0, #0 #ifdef HARVARD_CACHE mcr p15, 0, r0, c7, c14, 0 @ D cache clean+invalidate @@ -71,6 +74,7 @@ ENTRY(v6_flush_kern_cache_all) mcr p15, 0, r0, c7, c15, 0 @ Cache clean+invalidate #endif ret lr +SYM_FUNC_END(v6_flush_kern_cache_all) /* * v6_flush_cache_all() @@ -79,8 +83,9 @@ ENTRY(v6_flush_kern_cache_all) * * - mm - mm_struct describing address space */ -ENTRY(v6_flush_user_cache_all) - /*FALLTHROUGH*/ +SYM_TYPED_FUNC_START(v6_flush_user_cache_all) + ret lr +SYM_FUNC_END(v6_flush_user_cache_all) /* * v6_flush_cache_range(start, end, flags) @@ -94,8 +99,9 @@ ENTRY(v6_flush_user_cache_all) * It is assumed that: * - we have a VIPT cache. */ -ENTRY(v6_flush_user_cache_range) +SYM_TYPED_FUNC_START(v6_flush_user_cache_range) ret lr +SYM_FUNC_END(v6_flush_user_cache_range) /* * v6_coherent_kern_range(start,end) @@ -110,8 +116,11 @@ ENTRY(v6_flush_user_cache_range) * It is assumed that: * - the Icache does not read data from the write buffer */ -ENTRY(v6_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(v6_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b v6_coherent_user_range +#endif +SYM_FUNC_END(v6_coherent_kern_range) /* * v6_coherent_user_range(start,end) @@ -126,7 +135,7 @@ ENTRY(v6_coherent_kern_range) * It is assumed that: * - the Icache does not read data from the write buffer */ -ENTRY(v6_coherent_user_range) +SYM_TYPED_FUNC_START(v6_coherent_user_range) UNWIND(.fnstart ) #ifdef HARVARD_CACHE bic r0, r0, #CACHE_LINE_SIZE - 1 @@ -157,8 +166,7 @@ ENTRY(v6_coherent_user_range) mov r0, #-EFAULT ret lr UNWIND(.fnend ) -ENDPROC(v6_coherent_user_range) -ENDPROC(v6_coherent_kern_range) +SYM_FUNC_END(v6_coherent_user_range) /* * v6_flush_kern_dcache_area(void *addr, size_t size) @@ -169,7 +177,7 @@ ENDPROC(v6_coherent_kern_range) * - addr - kernel address * - size - region size */ -ENTRY(v6_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(v6_flush_kern_dcache_area) add r1, r0, r1 bic r0, r0, #D_CACHE_LINE_SIZE - 1 1: @@ -186,7 +194,7 @@ ENTRY(v6_flush_kern_dcache_area) mcr p15, 0, r0, c7, c10, 4 #endif ret lr - +SYM_FUNC_END(v6_flush_kern_dcache_area) /* * v6_dma_inv_range(start,end) @@ -199,10 +207,6 @@ ENTRY(v6_flush_kern_dcache_area) * - end - virtual end address of region */ v6_dma_inv_range: -#ifdef CONFIG_DMA_CACHE_RWFO - ldrb r2, [r0] @ read for ownership - strb r2, [r0] @ write for ownership -#endif tst r0, #D_CACHE_LINE_SIZE - 1 bic r0, r0, #D_CACHE_LINE_SIZE - 1 #ifdef HARVARD_CACHE @@ -211,10 +215,6 @@ v6_dma_inv_range: mcrne p15, 0, r0, c7, c11, 1 @ clean unified line #endif tst r1, #D_CACHE_LINE_SIZE - 1 -#ifdef CONFIG_DMA_CACHE_RWFO - ldrbne r2, [r1, #-1] @ read for ownership - strbne r2, [r1, #-1] @ write for ownership -#endif bic r1, r1, #D_CACHE_LINE_SIZE - 1 #ifdef HARVARD_CACHE mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D line @@ -229,10 +229,6 @@ v6_dma_inv_range: #endif add r0, r0, #D_CACHE_LINE_SIZE cmp r0, r1 -#ifdef CONFIG_DMA_CACHE_RWFO - ldrlo r2, [r0] @ read for ownership - strlo r2, [r0] @ write for ownership -#endif blo 1b mov r0, #0 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer @@ -246,9 +242,6 @@ v6_dma_inv_range: v6_dma_clean_range: bic r0, r0, #D_CACHE_LINE_SIZE - 1 1: -#ifdef CONFIG_DMA_CACHE_RWFO - ldr r2, [r0] @ read for ownership -#endif #ifdef HARVARD_CACHE mcr p15, 0, r0, c7, c10, 1 @ clean D line #else @@ -266,11 +259,7 @@ v6_dma_clean_range: * - start - virtual start address of region * - end - virtual end address of region */ -ENTRY(v6_dma_flush_range) -#ifdef CONFIG_DMA_CACHE_RWFO - ldrb r2, [r0] @ read for ownership - strb r2, [r0] @ write for ownership -#endif +SYM_TYPED_FUNC_START(v6_dma_flush_range) bic r0, r0, #D_CACHE_LINE_SIZE - 1 1: #ifdef HARVARD_CACHE @@ -280,14 +269,11 @@ ENTRY(v6_dma_flush_range) #endif add r0, r0, #D_CACHE_LINE_SIZE cmp r0, r1 -#ifdef CONFIG_DMA_CACHE_RWFO - ldrblo r2, [r0] @ read for ownership - strblo r2, [r0] @ write for ownership -#endif blo 1b mov r0, #0 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer ret lr +SYM_FUNC_END(v6_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -295,18 +281,12 @@ ENTRY(v6_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(v6_dma_map_area) +SYM_TYPED_FUNC_START(v6_dma_map_area) add r1, r1, r0 teq r2, #DMA_FROM_DEVICE beq v6_dma_inv_range -#ifndef CONFIG_DMA_CACHE_RWFO b v6_dma_clean_range -#else - teq r2, #DMA_TO_DEVICE - beq v6_dma_clean_range - b v6_dma_flush_range -#endif -ENDPROC(v6_dma_map_area) +SYM_FUNC_END(v6_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -314,19 +294,9 @@ ENDPROC(v6_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(v6_dma_unmap_area) -#ifndef CONFIG_DMA_CACHE_RWFO +SYM_TYPED_FUNC_START(v6_dma_unmap_area) add r1, r1, r0 teq r2, #DMA_TO_DEVICE bne v6_dma_inv_range -#endif ret lr -ENDPROC(v6_dma_unmap_area) - - .globl v6_flush_kern_cache_louis - .equ v6_flush_kern_cache_louis, v6_flush_kern_cache_all - - __INITDATA - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions v6 +SYM_FUNC_END(v6_dma_unmap_area) diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S index 0ee8fc4b4672..201ca05436fa 100644 --- a/arch/arm/mm/cache-v7.S +++ b/arch/arm/mm/cache-v7.S @@ -9,6 +9,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/errno.h> #include <asm/unwind.h> @@ -16,6 +17,8 @@ #include "proc-macros.S" +.arch armv7-a + #ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND .globl icache_size .data @@ -33,41 +36,41 @@ icache_size: * processor. We fix this by performing an invalidate, rather than a * clean + invalidate, before jumping into the kernel. * - * This function is cloned from arch/arm/mach-tegra/headsmp.S, and needs - * to be called for both secondary cores startup and primary core resume - * procedures. + * This function needs to be called for both secondary cores startup and + * primary core resume procedures. */ ENTRY(v7_invalidate_l1) - mov r0, #0 - mcr p15, 2, r0, c0, c0, 0 - mrc p15, 1, r0, c0, c0, 0 - - movw r1, #0x7fff - and r2, r1, r0, lsr #13 + mov r0, #0 + mcr p15, 2, r0, c0, c0, 0 @ select L1 data cache in CSSELR + isb + mrc p15, 1, r0, c0, c0, 0 @ read cache geometry from CCSIDR - movw r1, #0x3ff + movw r3, #0x3ff + and r3, r3, r0, lsr #3 @ 'Associativity' in CCSIDR[12:3] + clz r1, r3 @ WayShift + mov r2, #1 + mov r3, r3, lsl r1 @ NumWays-1 shifted into bits [31:...] + movs r1, r2, lsl r1 @ #1 shifted left by same amount + moveq r1, #1 @ r1 needs value > 0 even if only 1 way - and r3, r1, r0, lsr #3 @ NumWays - 1 - add r2, r2, #1 @ NumSets + and r2, r0, #0x7 + add r2, r2, #4 @ SetShift - and r0, r0, #0x7 - add r0, r0, #4 @ SetShift +1: movw ip, #0x7fff + and r0, ip, r0, lsr #13 @ 'NumSets' in CCSIDR[27:13] - clz r1, r3 @ WayShift - add r4, r3, #1 @ NumWays -1: sub r2, r2, #1 @ NumSets-- - mov r3, r4 @ Temp = NumWays -2: subs r3, r3, #1 @ Temp-- - mov r5, r3, lsl r1 - mov r6, r2, lsl r0 - orr r5, r5, r6 @ Reg = (Temp<<WayShift)|(NumSets<<SetShift) - mcr p15, 0, r5, c7, c6, 2 - bgt 2b - cmp r2, #0 - bgt 1b - dsb st - isb - ret lr +2: mov ip, r0, lsl r2 @ NumSet << SetShift + orr ip, ip, r3 @ Reg = (Temp<<WayShift)|(NumSets<<SetShift) + mcr p15, 0, ip, c7, c6, 2 + subs r0, r0, #1 @ Set-- + bpl 2b + subs r3, r3, r1 @ Way-- + bcc 3f + mrc p15, 1, r0, c0, c0, 0 @ re-read cache geometry from CCSIDR + b 1b +3: dsb st + isb + ret lr ENDPROC(v7_invalidate_l1) /* @@ -78,19 +81,19 @@ ENDPROC(v7_invalidate_l1) * Registers: * r0 - set to 0 */ -ENTRY(v7_flush_icache_all) +SYM_TYPED_FUNC_START(v7_flush_icache_all) mov r0, #0 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate ret lr -ENDPROC(v7_flush_icache_all) +SYM_FUNC_END(v7_flush_icache_all) /* * v7_flush_dcache_louis() * * Flush the D-cache up to the Level of Unification Inner Shareable * - * Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode) + * Corrupted registers: r0-r6, r9-r10 */ ENTRY(v7_flush_dcache_louis) @@ -117,7 +120,7 @@ ENDPROC(v7_flush_dcache_louis) * * Flush the whole D-cache. * - * Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode) + * Corrupted registers: r0-r6, r9-r10 * * - mm - mm_struct describing address space */ @@ -135,13 +138,13 @@ flush_levels: and r1, r1, #7 @ mask of the bits for current cache only cmp r1, #2 @ see what cache we have at this level blt skip @ skip if no cache, or just i-cache -#ifdef CONFIG_PREEMPT +#ifdef CONFIG_PREEMPTION save_and_disable_irqs_notrace r9 @ make cssr&csidr read atomic #endif mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr isb @ isb to sych the new cssr&csidr mrc p15, 1, r1, c0, c0, 0 @ read the new csidr -#ifdef CONFIG_PREEMPT +#ifdef CONFIG_PREEMPTION restore_irqs_notrace r9 #endif and r2, r1, #7 @ extract the length of the cache lines @@ -149,22 +152,22 @@ flush_levels: movw r4, #0x3ff ands r4, r4, r1, lsr #3 @ find maximum number on the way size clz r5, r4 @ find bit position of way size increment - movw r7, #0x7fff - ands r7, r7, r1, lsr #13 @ extract max number of the index size + movw r6, #0x7fff + and r1, r6, r1, lsr #13 @ extract max number of the index size + mov r6, #1 + movne r4, r4, lsl r5 @ # of ways shifted into bits [31:...] + movne r6, r6, lsl r5 @ 1 shifted left by same amount loop1: - mov r9, r7 @ create working copy of max index + mov r9, r1 @ create working copy of max index loop2: - ARM( orr r11, r10, r4, lsl r5 ) @ factor way and cache number into r11 - THUMB( lsl r6, r4, r5 ) - THUMB( orr r11, r10, r6 ) @ factor way and cache number into r11 - ARM( orr r11, r11, r9, lsl r2 ) @ factor index number into r11 - THUMB( lsl r6, r9, r2 ) - THUMB( orr r11, r11, r6 ) @ factor index number into r11 - mcr p15, 0, r11, c7, c14, 2 @ clean & invalidate by set/way + mov r5, r9, lsl r2 @ factor set number into r5 + orr r5, r5, r4 @ factor way number into r5 + orr r5, r5, r10 @ factor cache level into r5 + mcr p15, 0, r5, c7, c14, 2 @ clean & invalidate by set/way subs r9, r9, #1 @ decrement the index bge loop2 - subs r4, r4, #1 @ decrement the way - bge loop1 + subs r4, r4, r6 @ decrement the way + bcs loop1 skip: add r10, r10, #2 @ increment cache number cmp r3, r10 @@ -191,17 +194,15 @@ ENDPROC(v7_flush_dcache_all) * unification in a single instruction. * */ -ENTRY(v7_flush_kern_cache_all) - ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} ) - THUMB( stmfd sp!, {r4-r7, r9-r11, lr} ) +SYM_TYPED_FUNC_START(v7_flush_kern_cache_all) + stmfd sp!, {r4-r6, r9-r10, lr} bl v7_flush_dcache_all mov r0, #0 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate - ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} ) - THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} ) + ldmfd sp!, {r4-r6, r9-r10, lr} ret lr -ENDPROC(v7_flush_kern_cache_all) +SYM_FUNC_END(v7_flush_kern_cache_all) /* * v7_flush_kern_cache_louis(void) @@ -209,17 +210,15 @@ ENDPROC(v7_flush_kern_cache_all) * Flush the data cache up to Level of Unification Inner Shareable. * Invalidate the I-cache to the point of unification. */ -ENTRY(v7_flush_kern_cache_louis) - ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} ) - THUMB( stmfd sp!, {r4-r7, r9-r11, lr} ) +SYM_TYPED_FUNC_START(v7_flush_kern_cache_louis) + stmfd sp!, {r4-r6, r9-r10, lr} bl v7_flush_dcache_louis mov r0, #0 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate - ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} ) - THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} ) + ldmfd sp!, {r4-r6, r9-r10, lr} ret lr -ENDPROC(v7_flush_kern_cache_louis) +SYM_FUNC_END(v7_flush_kern_cache_louis) /* * v7_flush_cache_all() @@ -228,8 +227,9 @@ ENDPROC(v7_flush_kern_cache_louis) * * - mm - mm_struct describing address space */ -ENTRY(v7_flush_user_cache_all) - /*FALLTHROUGH*/ +SYM_TYPED_FUNC_START(v7_flush_user_cache_all) + ret lr +SYM_FUNC_END(v7_flush_user_cache_all) /* * v7_flush_cache_range(start, end, flags) @@ -243,10 +243,9 @@ ENTRY(v7_flush_user_cache_all) * It is assumed that: * - we have a VIPT cache. */ -ENTRY(v7_flush_user_cache_range) +SYM_TYPED_FUNC_START(v7_flush_user_cache_range) ret lr -ENDPROC(v7_flush_user_cache_all) -ENDPROC(v7_flush_user_cache_range) +SYM_FUNC_END(v7_flush_user_cache_range) /* * v7_coherent_kern_range(start,end) @@ -261,8 +260,11 @@ ENDPROC(v7_flush_user_cache_range) * It is assumed that: * - the Icache does not read data from the write buffer */ -ENTRY(v7_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(v7_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b v7_coherent_user_range +#endif +SYM_FUNC_END(v7_coherent_kern_range) /* * v7_coherent_user_range(start,end) @@ -277,7 +279,7 @@ ENTRY(v7_coherent_kern_range) * It is assumed that: * - the Icache does not read data from the write buffer */ -ENTRY(v7_coherent_user_range) +SYM_TYPED_FUNC_START(v7_coherent_user_range) UNWIND(.fnstart ) dcache_line_size r2, r3 sub r3, r2, #1 @@ -323,8 +325,7 @@ ENTRY(v7_coherent_user_range) mov r0, #-EFAULT ret lr UNWIND(.fnend ) -ENDPROC(v7_coherent_kern_range) -ENDPROC(v7_coherent_user_range) +SYM_FUNC_END(v7_coherent_user_range) /* * v7_flush_kern_dcache_area(void *addr, size_t size) @@ -335,7 +336,7 @@ ENDPROC(v7_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(v7_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(v7_flush_kern_dcache_area) dcache_line_size r2, r3 add r1, r0, r1 sub r3, r2, #1 @@ -351,7 +352,7 @@ ENTRY(v7_flush_kern_dcache_area) blo 1b dsb st ret lr -ENDPROC(v7_flush_kern_dcache_area) +SYM_FUNC_END(v7_flush_kern_dcache_area) /* * v7_dma_inv_range(start,end) @@ -415,7 +416,7 @@ ENDPROC(v7_dma_clean_range) * - start - virtual start address of region * - end - virtual end address of region */ -ENTRY(v7_dma_flush_range) +SYM_TYPED_FUNC_START(v7_dma_flush_range) dcache_line_size r2, r3 sub r3, r2, #1 bic r0, r0, r3 @@ -430,7 +431,7 @@ ENTRY(v7_dma_flush_range) blo 1b dsb st ret lr -ENDPROC(v7_dma_flush_range) +SYM_FUNC_END(v7_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -438,12 +439,12 @@ ENDPROC(v7_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(v7_dma_map_area) +SYM_TYPED_FUNC_START(v7_dma_map_area) add r1, r1, r0 teq r2, #DMA_FROM_DEVICE beq v7_dma_inv_range b v7_dma_clean_range -ENDPROC(v7_dma_map_area) +SYM_FUNC_END(v7_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -451,34 +452,9 @@ ENDPROC(v7_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(v7_dma_unmap_area) +SYM_TYPED_FUNC_START(v7_dma_unmap_area) add r1, r1, r0 teq r2, #DMA_TO_DEVICE bne v7_dma_inv_range ret lr -ENDPROC(v7_dma_unmap_area) - - __INITDATA - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions v7 - - /* The Broadcom Brahma-B15 read-ahead cache requires some modifications - * to the v7_cache_fns, we only override the ones we need - */ -#ifndef CONFIG_CACHE_B15_RAC - globl_equ b15_flush_kern_cache_all, v7_flush_kern_cache_all -#endif - globl_equ b15_flush_icache_all, v7_flush_icache_all - globl_equ b15_flush_kern_cache_louis, v7_flush_kern_cache_louis - globl_equ b15_flush_user_cache_all, v7_flush_user_cache_all - globl_equ b15_flush_user_cache_range, v7_flush_user_cache_range - globl_equ b15_coherent_kern_range, v7_coherent_kern_range - globl_equ b15_coherent_user_range, v7_coherent_user_range - globl_equ b15_flush_kern_dcache_area, v7_flush_kern_dcache_area - - globl_equ b15_dma_map_area, v7_dma_map_area - globl_equ b15_dma_unmap_area, v7_dma_unmap_area - globl_equ b15_dma_flush_range, v7_dma_flush_range - - define_cache_functions b15 +SYM_FUNC_END(v7_dma_unmap_area) diff --git a/arch/arm/mm/cache-v7m.S b/arch/arm/mm/cache-v7m.S index a0035c426ce6..14d719eba729 100644 --- a/arch/arm/mm/cache-v7m.S +++ b/arch/arm/mm/cache-v7m.S @@ -11,6 +11,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/errno.h> #include <asm/unwind.h> @@ -18,6 +19,8 @@ #include "proc-macros.S" +.arch armv7-m + /* Generic V7M read/write macros for memory mapped cache operations */ .macro v7m_cache_read, rt, reg movw \rt, #:lower16:BASEADDR_V7M_SCB + \reg @@ -157,10 +160,10 @@ ENDPROC(v7m_invalidate_l1) * Registers: * r0 - set to 0 */ -ENTRY(v7m_flush_icache_all) +SYM_TYPED_FUNC_START(v7m_flush_icache_all) invalidate_icache r0 ret lr -ENDPROC(v7m_flush_icache_all) +SYM_FUNC_END(v7m_flush_icache_all) /* * v7m_flush_dcache_all() @@ -183,13 +186,13 @@ flush_levels: and r1, r1, #7 @ mask of the bits for current cache only cmp r1, #2 @ see what cache we have at this level blt skip @ skip if no cache, or just i-cache -#ifdef CONFIG_PREEMPT +#ifdef CONFIG_PREEMPTION save_and_disable_irqs_notrace r9 @ make cssr&csidr read atomic #endif write_csselr r10, r1 @ set current cache level isb @ isb to sych the new cssr&csidr read_ccsidr r1 @ read the new csidr -#ifdef CONFIG_PREEMPT +#ifdef CONFIG_PREEMPTION restore_irqs_notrace r9 #endif and r2, r1, #7 @ extract the length of the cache lines @@ -234,13 +237,13 @@ ENDPROC(v7m_flush_dcache_all) * unification in a single instruction. * */ -ENTRY(v7m_flush_kern_cache_all) +SYM_TYPED_FUNC_START(v7m_flush_kern_cache_all) stmfd sp!, {r4-r7, r9-r11, lr} bl v7m_flush_dcache_all invalidate_icache r0 ldmfd sp!, {r4-r7, r9-r11, lr} ret lr -ENDPROC(v7m_flush_kern_cache_all) +SYM_FUNC_END(v7m_flush_kern_cache_all) /* * v7m_flush_cache_all() @@ -249,8 +252,9 @@ ENDPROC(v7m_flush_kern_cache_all) * * - mm - mm_struct describing address space */ -ENTRY(v7m_flush_user_cache_all) - /*FALLTHROUGH*/ +SYM_TYPED_FUNC_START(v7m_flush_user_cache_all) + ret lr +SYM_FUNC_END(v7m_flush_user_cache_all) /* * v7m_flush_cache_range(start, end, flags) @@ -264,10 +268,9 @@ ENTRY(v7m_flush_user_cache_all) * It is assumed that: * - we have a VIPT cache. */ -ENTRY(v7m_flush_user_cache_range) +SYM_TYPED_FUNC_START(v7m_flush_user_cache_range) ret lr -ENDPROC(v7m_flush_user_cache_all) -ENDPROC(v7m_flush_user_cache_range) +SYM_FUNC_END(v7m_flush_user_cache_range) /* * v7m_coherent_kern_range(start,end) @@ -282,8 +285,11 @@ ENDPROC(v7m_flush_user_cache_range) * It is assumed that: * - the Icache does not read data from the write buffer */ -ENTRY(v7m_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(v7m_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b v7m_coherent_user_range +#endif +SYM_FUNC_END(v7m_coherent_kern_range) /* * v7m_coherent_user_range(start,end) @@ -298,7 +304,7 @@ ENTRY(v7m_coherent_kern_range) * It is assumed that: * - the Icache does not read data from the write buffer */ -ENTRY(v7m_coherent_user_range) +SYM_TYPED_FUNC_START(v7m_coherent_user_range) UNWIND(.fnstart ) dcache_line_size r2, r3 sub r3, r2, #1 @@ -326,8 +332,7 @@ ENTRY(v7m_coherent_user_range) isb ret lr UNWIND(.fnend ) -ENDPROC(v7m_coherent_kern_range) -ENDPROC(v7m_coherent_user_range) +SYM_FUNC_END(v7m_coherent_user_range) /* * v7m_flush_kern_dcache_area(void *addr, size_t size) @@ -338,7 +343,7 @@ ENDPROC(v7m_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(v7m_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(v7m_flush_kern_dcache_area) dcache_line_size r2, r3 add r1, r0, r1 sub r3, r2, #1 @@ -350,7 +355,7 @@ ENTRY(v7m_flush_kern_dcache_area) blo 1b dsb st ret lr -ENDPROC(v7m_flush_kern_dcache_area) +SYM_FUNC_END(v7m_flush_kern_dcache_area) /* * v7m_dma_inv_range(start,end) @@ -406,7 +411,7 @@ ENDPROC(v7m_dma_clean_range) * - start - virtual start address of region * - end - virtual end address of region */ -ENTRY(v7m_dma_flush_range) +SYM_TYPED_FUNC_START(v7m_dma_flush_range) dcache_line_size r2, r3 sub r3, r2, #1 bic r0, r0, r3 @@ -417,7 +422,7 @@ ENTRY(v7m_dma_flush_range) blo 1b dsb st ret lr -ENDPROC(v7m_dma_flush_range) +SYM_FUNC_END(v7m_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -425,12 +430,12 @@ ENDPROC(v7m_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(v7m_dma_map_area) +SYM_TYPED_FUNC_START(v7m_dma_map_area) add r1, r1, r0 teq r2, #DMA_FROM_DEVICE beq v7m_dma_inv_range b v7m_dma_clean_range -ENDPROC(v7m_dma_map_area) +SYM_FUNC_END(v7m_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -438,17 +443,9 @@ ENDPROC(v7m_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(v7m_dma_unmap_area) +SYM_TYPED_FUNC_START(v7m_dma_unmap_area) add r1, r1, r0 teq r2, #DMA_TO_DEVICE bne v7m_dma_inv_range ret lr -ENDPROC(v7m_dma_unmap_area) - - .globl v7m_flush_kern_cache_louis - .equ v7m_flush_kern_cache_louis, v7m_flush_kern_cache_all - - __INITDATA - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions v7m +SYM_FUNC_END(v7m_dma_unmap_area) diff --git a/arch/arm/mm/cache.c b/arch/arm/mm/cache.c new file mode 100644 index 000000000000..e6fbc599c9ed --- /dev/null +++ b/arch/arm/mm/cache.c @@ -0,0 +1,663 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file defines C prototypes for the low-level cache assembly functions + * and populates a vtable for each selected ARM CPU cache type. + */ + +#include <linux/types.h> +#include <asm/cacheflush.h> + +#ifdef CONFIG_CPU_CACHE_V4 +void v4_flush_icache_all(void); +void v4_flush_kern_cache_all(void); +void v4_flush_user_cache_all(void); +void v4_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void v4_coherent_kern_range(unsigned long, unsigned long); +int v4_coherent_user_range(unsigned long, unsigned long); +void v4_flush_kern_dcache_area(void *, size_t); +void v4_dma_map_area(const void *, size_t, int); +void v4_dma_unmap_area(const void *, size_t, int); +void v4_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns v4_cache_fns __initconst = { + .flush_icache_all = v4_flush_icache_all, + .flush_kern_all = v4_flush_kern_cache_all, + .flush_kern_louis = v4_flush_kern_cache_all, + .flush_user_all = v4_flush_user_cache_all, + .flush_user_range = v4_flush_user_cache_range, + .coherent_kern_range = v4_coherent_kern_range, + .coherent_user_range = v4_coherent_user_range, + .flush_kern_dcache_area = v4_flush_kern_dcache_area, + .dma_map_area = v4_dma_map_area, + .dma_unmap_area = v4_dma_unmap_area, + .dma_flush_range = v4_dma_flush_range, +}; +#endif + +/* V4 write-back cache "V4WB" */ +#ifdef CONFIG_CPU_CACHE_V4WB +void v4wb_flush_icache_all(void); +void v4wb_flush_kern_cache_all(void); +void v4wb_flush_user_cache_all(void); +void v4wb_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void v4wb_coherent_kern_range(unsigned long, unsigned long); +int v4wb_coherent_user_range(unsigned long, unsigned long); +void v4wb_flush_kern_dcache_area(void *, size_t); +void v4wb_dma_map_area(const void *, size_t, int); +void v4wb_dma_unmap_area(const void *, size_t, int); +void v4wb_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns v4wb_cache_fns __initconst = { + .flush_icache_all = v4wb_flush_icache_all, + .flush_kern_all = v4wb_flush_kern_cache_all, + .flush_kern_louis = v4wb_flush_kern_cache_all, + .flush_user_all = v4wb_flush_user_cache_all, + .flush_user_range = v4wb_flush_user_cache_range, + .coherent_kern_range = v4wb_coherent_kern_range, + .coherent_user_range = v4wb_coherent_user_range, + .flush_kern_dcache_area = v4wb_flush_kern_dcache_area, + .dma_map_area = v4wb_dma_map_area, + .dma_unmap_area = v4wb_dma_unmap_area, + .dma_flush_range = v4wb_dma_flush_range, +}; +#endif + +/* V4 write-through cache "V4WT" */ +#ifdef CONFIG_CPU_CACHE_V4WT +void v4wt_flush_icache_all(void); +void v4wt_flush_kern_cache_all(void); +void v4wt_flush_user_cache_all(void); +void v4wt_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void v4wt_coherent_kern_range(unsigned long, unsigned long); +int v4wt_coherent_user_range(unsigned long, unsigned long); +void v4wt_flush_kern_dcache_area(void *, size_t); +void v4wt_dma_map_area(const void *, size_t, int); +void v4wt_dma_unmap_area(const void *, size_t, int); +void v4wt_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns v4wt_cache_fns __initconst = { + .flush_icache_all = v4wt_flush_icache_all, + .flush_kern_all = v4wt_flush_kern_cache_all, + .flush_kern_louis = v4wt_flush_kern_cache_all, + .flush_user_all = v4wt_flush_user_cache_all, + .flush_user_range = v4wt_flush_user_cache_range, + .coherent_kern_range = v4wt_coherent_kern_range, + .coherent_user_range = v4wt_coherent_user_range, + .flush_kern_dcache_area = v4wt_flush_kern_dcache_area, + .dma_map_area = v4wt_dma_map_area, + .dma_unmap_area = v4wt_dma_unmap_area, + .dma_flush_range = v4wt_dma_flush_range, +}; +#endif + +/* Faraday FA526 cache */ +#ifdef CONFIG_CPU_CACHE_FA +void fa_flush_icache_all(void); +void fa_flush_kern_cache_all(void); +void fa_flush_user_cache_all(void); +void fa_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void fa_coherent_kern_range(unsigned long, unsigned long); +int fa_coherent_user_range(unsigned long, unsigned long); +void fa_flush_kern_dcache_area(void *, size_t); +void fa_dma_map_area(const void *, size_t, int); +void fa_dma_unmap_area(const void *, size_t, int); +void fa_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns fa_cache_fns __initconst = { + .flush_icache_all = fa_flush_icache_all, + .flush_kern_all = fa_flush_kern_cache_all, + .flush_kern_louis = fa_flush_kern_cache_all, + .flush_user_all = fa_flush_user_cache_all, + .flush_user_range = fa_flush_user_cache_range, + .coherent_kern_range = fa_coherent_kern_range, + .coherent_user_range = fa_coherent_user_range, + .flush_kern_dcache_area = fa_flush_kern_dcache_area, + .dma_map_area = fa_dma_map_area, + .dma_unmap_area = fa_dma_unmap_area, + .dma_flush_range = fa_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_CACHE_V6 +void v6_flush_icache_all(void); +void v6_flush_kern_cache_all(void); +void v6_flush_user_cache_all(void); +void v6_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void v6_coherent_kern_range(unsigned long, unsigned long); +int v6_coherent_user_range(unsigned long, unsigned long); +void v6_flush_kern_dcache_area(void *, size_t); +void v6_dma_map_area(const void *, size_t, int); +void v6_dma_unmap_area(const void *, size_t, int); +void v6_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns v6_cache_fns __initconst = { + .flush_icache_all = v6_flush_icache_all, + .flush_kern_all = v6_flush_kern_cache_all, + .flush_kern_louis = v6_flush_kern_cache_all, + .flush_user_all = v6_flush_user_cache_all, + .flush_user_range = v6_flush_user_cache_range, + .coherent_kern_range = v6_coherent_kern_range, + .coherent_user_range = v6_coherent_user_range, + .flush_kern_dcache_area = v6_flush_kern_dcache_area, + .dma_map_area = v6_dma_map_area, + .dma_unmap_area = v6_dma_unmap_area, + .dma_flush_range = v6_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_CACHE_V7 +void v7_flush_icache_all(void); +void v7_flush_kern_cache_all(void); +void v7_flush_kern_cache_louis(void); +void v7_flush_user_cache_all(void); +void v7_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void v7_coherent_kern_range(unsigned long, unsigned long); +int v7_coherent_user_range(unsigned long, unsigned long); +void v7_flush_kern_dcache_area(void *, size_t); +void v7_dma_map_area(const void *, size_t, int); +void v7_dma_unmap_area(const void *, size_t, int); +void v7_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns v7_cache_fns __initconst = { + .flush_icache_all = v7_flush_icache_all, + .flush_kern_all = v7_flush_kern_cache_all, + .flush_kern_louis = v7_flush_kern_cache_louis, + .flush_user_all = v7_flush_user_cache_all, + .flush_user_range = v7_flush_user_cache_range, + .coherent_kern_range = v7_coherent_kern_range, + .coherent_user_range = v7_coherent_user_range, + .flush_kern_dcache_area = v7_flush_kern_dcache_area, + .dma_map_area = v7_dma_map_area, + .dma_unmap_area = v7_dma_unmap_area, + .dma_flush_range = v7_dma_flush_range, +}; + +/* Special quirky cache flush function for Broadcom B15 v7 caches */ +void b15_flush_kern_cache_all(void); + +struct cpu_cache_fns b15_cache_fns __initconst = { + .flush_icache_all = v7_flush_icache_all, +#ifdef CONFIG_CACHE_B15_RAC + .flush_kern_all = b15_flush_kern_cache_all, +#else + .flush_kern_all = v7_flush_kern_cache_all, +#endif + .flush_kern_louis = v7_flush_kern_cache_louis, + .flush_user_all = v7_flush_user_cache_all, + .flush_user_range = v7_flush_user_cache_range, + .coherent_kern_range = v7_coherent_kern_range, + .coherent_user_range = v7_coherent_user_range, + .flush_kern_dcache_area = v7_flush_kern_dcache_area, + .dma_map_area = v7_dma_map_area, + .dma_unmap_area = v7_dma_unmap_area, + .dma_flush_range = v7_dma_flush_range, +}; +#endif + +/* The NOP cache is just a set of dummy stubs that by definition does nothing */ +#ifdef CONFIG_CPU_CACHE_NOP +void nop_flush_icache_all(void); +void nop_flush_kern_cache_all(void); +void nop_flush_user_cache_all(void); +void nop_flush_user_cache_range(unsigned long start, unsigned long end, unsigned int flags); +void nop_coherent_kern_range(unsigned long start, unsigned long end); +int nop_coherent_user_range(unsigned long, unsigned long); +void nop_flush_kern_dcache_area(void *kaddr, size_t size); +void nop_dma_map_area(const void *start, size_t size, int flags); +void nop_dma_unmap_area(const void *start, size_t size, int flags); +void nop_dma_flush_range(const void *start, const void *end); + +struct cpu_cache_fns nop_cache_fns __initconst = { + .flush_icache_all = nop_flush_icache_all, + .flush_kern_all = nop_flush_kern_cache_all, + .flush_kern_louis = nop_flush_kern_cache_all, + .flush_user_all = nop_flush_user_cache_all, + .flush_user_range = nop_flush_user_cache_range, + .coherent_kern_range = nop_coherent_kern_range, + .coherent_user_range = nop_coherent_user_range, + .flush_kern_dcache_area = nop_flush_kern_dcache_area, + .dma_map_area = nop_dma_map_area, + .dma_unmap_area = nop_dma_unmap_area, + .dma_flush_range = nop_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_CACHE_V7M +void v7m_flush_icache_all(void); +void v7m_flush_kern_cache_all(void); +void v7m_flush_user_cache_all(void); +void v7m_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void v7m_coherent_kern_range(unsigned long, unsigned long); +int v7m_coherent_user_range(unsigned long, unsigned long); +void v7m_flush_kern_dcache_area(void *, size_t); +void v7m_dma_map_area(const void *, size_t, int); +void v7m_dma_unmap_area(const void *, size_t, int); +void v7m_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns v7m_cache_fns __initconst = { + .flush_icache_all = v7m_flush_icache_all, + .flush_kern_all = v7m_flush_kern_cache_all, + .flush_kern_louis = v7m_flush_kern_cache_all, + .flush_user_all = v7m_flush_user_cache_all, + .flush_user_range = v7m_flush_user_cache_range, + .coherent_kern_range = v7m_coherent_kern_range, + .coherent_user_range = v7m_coherent_user_range, + .flush_kern_dcache_area = v7m_flush_kern_dcache_area, + .dma_map_area = v7m_dma_map_area, + .dma_unmap_area = v7m_dma_unmap_area, + .dma_flush_range = v7m_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_ARM1020 +void arm1020_flush_icache_all(void); +void arm1020_flush_kern_cache_all(void); +void arm1020_flush_user_cache_all(void); +void arm1020_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void arm1020_coherent_kern_range(unsigned long, unsigned long); +int arm1020_coherent_user_range(unsigned long, unsigned long); +void arm1020_flush_kern_dcache_area(void *, size_t); +void arm1020_dma_map_area(const void *, size_t, int); +void arm1020_dma_unmap_area(const void *, size_t, int); +void arm1020_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns arm1020_cache_fns __initconst = { + .flush_icache_all = arm1020_flush_icache_all, + .flush_kern_all = arm1020_flush_kern_cache_all, + .flush_kern_louis = arm1020_flush_kern_cache_all, + .flush_user_all = arm1020_flush_user_cache_all, + .flush_user_range = arm1020_flush_user_cache_range, + .coherent_kern_range = arm1020_coherent_kern_range, + .coherent_user_range = arm1020_coherent_user_range, + .flush_kern_dcache_area = arm1020_flush_kern_dcache_area, + .dma_map_area = arm1020_dma_map_area, + .dma_unmap_area = arm1020_dma_unmap_area, + .dma_flush_range = arm1020_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_ARM1020E +void arm1020e_flush_icache_all(void); +void arm1020e_flush_kern_cache_all(void); +void arm1020e_flush_user_cache_all(void); +void arm1020e_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void arm1020e_coherent_kern_range(unsigned long, unsigned long); +int arm1020e_coherent_user_range(unsigned long, unsigned long); +void arm1020e_flush_kern_dcache_area(void *, size_t); +void arm1020e_dma_map_area(const void *, size_t, int); +void arm1020e_dma_unmap_area(const void *, size_t, int); +void arm1020e_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns arm1020e_cache_fns __initconst = { + .flush_icache_all = arm1020e_flush_icache_all, + .flush_kern_all = arm1020e_flush_kern_cache_all, + .flush_kern_louis = arm1020e_flush_kern_cache_all, + .flush_user_all = arm1020e_flush_user_cache_all, + .flush_user_range = arm1020e_flush_user_cache_range, + .coherent_kern_range = arm1020e_coherent_kern_range, + .coherent_user_range = arm1020e_coherent_user_range, + .flush_kern_dcache_area = arm1020e_flush_kern_dcache_area, + .dma_map_area = arm1020e_dma_map_area, + .dma_unmap_area = arm1020e_dma_unmap_area, + .dma_flush_range = arm1020e_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_ARM1022 +void arm1022_flush_icache_all(void); +void arm1022_flush_kern_cache_all(void); +void arm1022_flush_user_cache_all(void); +void arm1022_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void arm1022_coherent_kern_range(unsigned long, unsigned long); +int arm1022_coherent_user_range(unsigned long, unsigned long); +void arm1022_flush_kern_dcache_area(void *, size_t); +void arm1022_dma_map_area(const void *, size_t, int); +void arm1022_dma_unmap_area(const void *, size_t, int); +void arm1022_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns arm1022_cache_fns __initconst = { + .flush_icache_all = arm1022_flush_icache_all, + .flush_kern_all = arm1022_flush_kern_cache_all, + .flush_kern_louis = arm1022_flush_kern_cache_all, + .flush_user_all = arm1022_flush_user_cache_all, + .flush_user_range = arm1022_flush_user_cache_range, + .coherent_kern_range = arm1022_coherent_kern_range, + .coherent_user_range = arm1022_coherent_user_range, + .flush_kern_dcache_area = arm1022_flush_kern_dcache_area, + .dma_map_area = arm1022_dma_map_area, + .dma_unmap_area = arm1022_dma_unmap_area, + .dma_flush_range = arm1022_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_ARM1026 +void arm1026_flush_icache_all(void); +void arm1026_flush_kern_cache_all(void); +void arm1026_flush_user_cache_all(void); +void arm1026_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void arm1026_coherent_kern_range(unsigned long, unsigned long); +int arm1026_coherent_user_range(unsigned long, unsigned long); +void arm1026_flush_kern_dcache_area(void *, size_t); +void arm1026_dma_map_area(const void *, size_t, int); +void arm1026_dma_unmap_area(const void *, size_t, int); +void arm1026_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns arm1026_cache_fns __initconst = { + .flush_icache_all = arm1026_flush_icache_all, + .flush_kern_all = arm1026_flush_kern_cache_all, + .flush_kern_louis = arm1026_flush_kern_cache_all, + .flush_user_all = arm1026_flush_user_cache_all, + .flush_user_range = arm1026_flush_user_cache_range, + .coherent_kern_range = arm1026_coherent_kern_range, + .coherent_user_range = arm1026_coherent_user_range, + .flush_kern_dcache_area = arm1026_flush_kern_dcache_area, + .dma_map_area = arm1026_dma_map_area, + .dma_unmap_area = arm1026_dma_unmap_area, + .dma_flush_range = arm1026_dma_flush_range, +}; +#endif + +#if defined(CONFIG_CPU_ARM920T) && !defined(CONFIG_CPU_DCACHE_WRITETHROUGH) +void arm920_flush_icache_all(void); +void arm920_flush_kern_cache_all(void); +void arm920_flush_user_cache_all(void); +void arm920_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void arm920_coherent_kern_range(unsigned long, unsigned long); +int arm920_coherent_user_range(unsigned long, unsigned long); +void arm920_flush_kern_dcache_area(void *, size_t); +void arm920_dma_map_area(const void *, size_t, int); +void arm920_dma_unmap_area(const void *, size_t, int); +void arm920_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns arm920_cache_fns __initconst = { + .flush_icache_all = arm920_flush_icache_all, + .flush_kern_all = arm920_flush_kern_cache_all, + .flush_kern_louis = arm920_flush_kern_cache_all, + .flush_user_all = arm920_flush_user_cache_all, + .flush_user_range = arm920_flush_user_cache_range, + .coherent_kern_range = arm920_coherent_kern_range, + .coherent_user_range = arm920_coherent_user_range, + .flush_kern_dcache_area = arm920_flush_kern_dcache_area, + .dma_map_area = arm920_dma_map_area, + .dma_unmap_area = arm920_dma_unmap_area, + .dma_flush_range = arm920_dma_flush_range, +}; +#endif + +#if defined(CONFIG_CPU_ARM922T) && !defined(CONFIG_CPU_DCACHE_WRITETHROUGH) +void arm922_flush_icache_all(void); +void arm922_flush_kern_cache_all(void); +void arm922_flush_user_cache_all(void); +void arm922_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void arm922_coherent_kern_range(unsigned long, unsigned long); +int arm922_coherent_user_range(unsigned long, unsigned long); +void arm922_flush_kern_dcache_area(void *, size_t); +void arm922_dma_map_area(const void *, size_t, int); +void arm922_dma_unmap_area(const void *, size_t, int); +void arm922_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns arm922_cache_fns __initconst = { + .flush_icache_all = arm922_flush_icache_all, + .flush_kern_all = arm922_flush_kern_cache_all, + .flush_kern_louis = arm922_flush_kern_cache_all, + .flush_user_all = arm922_flush_user_cache_all, + .flush_user_range = arm922_flush_user_cache_range, + .coherent_kern_range = arm922_coherent_kern_range, + .coherent_user_range = arm922_coherent_user_range, + .flush_kern_dcache_area = arm922_flush_kern_dcache_area, + .dma_map_area = arm922_dma_map_area, + .dma_unmap_area = arm922_dma_unmap_area, + .dma_flush_range = arm922_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_ARM925T +void arm925_flush_icache_all(void); +void arm925_flush_kern_cache_all(void); +void arm925_flush_user_cache_all(void); +void arm925_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void arm925_coherent_kern_range(unsigned long, unsigned long); +int arm925_coherent_user_range(unsigned long, unsigned long); +void arm925_flush_kern_dcache_area(void *, size_t); +void arm925_dma_map_area(const void *, size_t, int); +void arm925_dma_unmap_area(const void *, size_t, int); +void arm925_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns arm925_cache_fns __initconst = { + .flush_icache_all = arm925_flush_icache_all, + .flush_kern_all = arm925_flush_kern_cache_all, + .flush_kern_louis = arm925_flush_kern_cache_all, + .flush_user_all = arm925_flush_user_cache_all, + .flush_user_range = arm925_flush_user_cache_range, + .coherent_kern_range = arm925_coherent_kern_range, + .coherent_user_range = arm925_coherent_user_range, + .flush_kern_dcache_area = arm925_flush_kern_dcache_area, + .dma_map_area = arm925_dma_map_area, + .dma_unmap_area = arm925_dma_unmap_area, + .dma_flush_range = arm925_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_ARM926T +void arm926_flush_icache_all(void); +void arm926_flush_kern_cache_all(void); +void arm926_flush_user_cache_all(void); +void arm926_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void arm926_coherent_kern_range(unsigned long, unsigned long); +int arm926_coherent_user_range(unsigned long, unsigned long); +void arm926_flush_kern_dcache_area(void *, size_t); +void arm926_dma_map_area(const void *, size_t, int); +void arm926_dma_unmap_area(const void *, size_t, int); +void arm926_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns arm926_cache_fns __initconst = { + .flush_icache_all = arm926_flush_icache_all, + .flush_kern_all = arm926_flush_kern_cache_all, + .flush_kern_louis = arm926_flush_kern_cache_all, + .flush_user_all = arm926_flush_user_cache_all, + .flush_user_range = arm926_flush_user_cache_range, + .coherent_kern_range = arm926_coherent_kern_range, + .coherent_user_range = arm926_coherent_user_range, + .flush_kern_dcache_area = arm926_flush_kern_dcache_area, + .dma_map_area = arm926_dma_map_area, + .dma_unmap_area = arm926_dma_unmap_area, + .dma_flush_range = arm926_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_ARM940T +void arm940_flush_icache_all(void); +void arm940_flush_kern_cache_all(void); +void arm940_flush_user_cache_all(void); +void arm940_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void arm940_coherent_kern_range(unsigned long, unsigned long); +int arm940_coherent_user_range(unsigned long, unsigned long); +void arm940_flush_kern_dcache_area(void *, size_t); +void arm940_dma_map_area(const void *, size_t, int); +void arm940_dma_unmap_area(const void *, size_t, int); +void arm940_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns arm940_cache_fns __initconst = { + .flush_icache_all = arm940_flush_icache_all, + .flush_kern_all = arm940_flush_kern_cache_all, + .flush_kern_louis = arm940_flush_kern_cache_all, + .flush_user_all = arm940_flush_user_cache_all, + .flush_user_range = arm940_flush_user_cache_range, + .coherent_kern_range = arm940_coherent_kern_range, + .coherent_user_range = arm940_coherent_user_range, + .flush_kern_dcache_area = arm940_flush_kern_dcache_area, + .dma_map_area = arm940_dma_map_area, + .dma_unmap_area = arm940_dma_unmap_area, + .dma_flush_range = arm940_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_ARM946E +void arm946_flush_icache_all(void); +void arm946_flush_kern_cache_all(void); +void arm946_flush_user_cache_all(void); +void arm946_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void arm946_coherent_kern_range(unsigned long, unsigned long); +int arm946_coherent_user_range(unsigned long, unsigned long); +void arm946_flush_kern_dcache_area(void *, size_t); +void arm946_dma_map_area(const void *, size_t, int); +void arm946_dma_unmap_area(const void *, size_t, int); +void arm946_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns arm946_cache_fns __initconst = { + .flush_icache_all = arm946_flush_icache_all, + .flush_kern_all = arm946_flush_kern_cache_all, + .flush_kern_louis = arm946_flush_kern_cache_all, + .flush_user_all = arm946_flush_user_cache_all, + .flush_user_range = arm946_flush_user_cache_range, + .coherent_kern_range = arm946_coherent_kern_range, + .coherent_user_range = arm946_coherent_user_range, + .flush_kern_dcache_area = arm946_flush_kern_dcache_area, + .dma_map_area = arm946_dma_map_area, + .dma_unmap_area = arm946_dma_unmap_area, + .dma_flush_range = arm946_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_XSCALE +void xscale_flush_icache_all(void); +void xscale_flush_kern_cache_all(void); +void xscale_flush_user_cache_all(void); +void xscale_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void xscale_coherent_kern_range(unsigned long, unsigned long); +int xscale_coherent_user_range(unsigned long, unsigned long); +void xscale_flush_kern_dcache_area(void *, size_t); +void xscale_dma_map_area(const void *, size_t, int); +void xscale_dma_unmap_area(const void *, size_t, int); +void xscale_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns xscale_cache_fns __initconst = { + .flush_icache_all = xscale_flush_icache_all, + .flush_kern_all = xscale_flush_kern_cache_all, + .flush_kern_louis = xscale_flush_kern_cache_all, + .flush_user_all = xscale_flush_user_cache_all, + .flush_user_range = xscale_flush_user_cache_range, + .coherent_kern_range = xscale_coherent_kern_range, + .coherent_user_range = xscale_coherent_user_range, + .flush_kern_dcache_area = xscale_flush_kern_dcache_area, + .dma_map_area = xscale_dma_map_area, + .dma_unmap_area = xscale_dma_unmap_area, + .dma_flush_range = xscale_dma_flush_range, +}; + +/* The 80200 A0 and A1 need a special quirk for dma_map_area() */ +void xscale_80200_A0_A1_dma_map_area(const void *, size_t, int); + +struct cpu_cache_fns xscale_80200_A0_A1_cache_fns __initconst = { + .flush_icache_all = xscale_flush_icache_all, + .flush_kern_all = xscale_flush_kern_cache_all, + .flush_kern_louis = xscale_flush_kern_cache_all, + .flush_user_all = xscale_flush_user_cache_all, + .flush_user_range = xscale_flush_user_cache_range, + .coherent_kern_range = xscale_coherent_kern_range, + .coherent_user_range = xscale_coherent_user_range, + .flush_kern_dcache_area = xscale_flush_kern_dcache_area, + .dma_map_area = xscale_80200_A0_A1_dma_map_area, + .dma_unmap_area = xscale_dma_unmap_area, + .dma_flush_range = xscale_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_XSC3 +void xsc3_flush_icache_all(void); +void xsc3_flush_kern_cache_all(void); +void xsc3_flush_user_cache_all(void); +void xsc3_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void xsc3_coherent_kern_range(unsigned long, unsigned long); +int xsc3_coherent_user_range(unsigned long, unsigned long); +void xsc3_flush_kern_dcache_area(void *, size_t); +void xsc3_dma_map_area(const void *, size_t, int); +void xsc3_dma_unmap_area(const void *, size_t, int); +void xsc3_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns xsc3_cache_fns __initconst = { + .flush_icache_all = xsc3_flush_icache_all, + .flush_kern_all = xsc3_flush_kern_cache_all, + .flush_kern_louis = xsc3_flush_kern_cache_all, + .flush_user_all = xsc3_flush_user_cache_all, + .flush_user_range = xsc3_flush_user_cache_range, + .coherent_kern_range = xsc3_coherent_kern_range, + .coherent_user_range = xsc3_coherent_user_range, + .flush_kern_dcache_area = xsc3_flush_kern_dcache_area, + .dma_map_area = xsc3_dma_map_area, + .dma_unmap_area = xsc3_dma_unmap_area, + .dma_flush_range = xsc3_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_MOHAWK +void mohawk_flush_icache_all(void); +void mohawk_flush_kern_cache_all(void); +void mohawk_flush_user_cache_all(void); +void mohawk_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void mohawk_coherent_kern_range(unsigned long, unsigned long); +int mohawk_coherent_user_range(unsigned long, unsigned long); +void mohawk_flush_kern_dcache_area(void *, size_t); +void mohawk_dma_map_area(const void *, size_t, int); +void mohawk_dma_unmap_area(const void *, size_t, int); +void mohawk_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns mohawk_cache_fns __initconst = { + .flush_icache_all = mohawk_flush_icache_all, + .flush_kern_all = mohawk_flush_kern_cache_all, + .flush_kern_louis = mohawk_flush_kern_cache_all, + .flush_user_all = mohawk_flush_user_cache_all, + .flush_user_range = mohawk_flush_user_cache_range, + .coherent_kern_range = mohawk_coherent_kern_range, + .coherent_user_range = mohawk_coherent_user_range, + .flush_kern_dcache_area = mohawk_flush_kern_dcache_area, + .dma_map_area = mohawk_dma_map_area, + .dma_unmap_area = mohawk_dma_unmap_area, + .dma_flush_range = mohawk_dma_flush_range, +}; +#endif + +#ifdef CONFIG_CPU_FEROCEON +void feroceon_flush_icache_all(void); +void feroceon_flush_kern_cache_all(void); +void feroceon_flush_user_cache_all(void); +void feroceon_flush_user_cache_range(unsigned long, unsigned long, unsigned int); +void feroceon_coherent_kern_range(unsigned long, unsigned long); +int feroceon_coherent_user_range(unsigned long, unsigned long); +void feroceon_flush_kern_dcache_area(void *, size_t); +void feroceon_dma_map_area(const void *, size_t, int); +void feroceon_dma_unmap_area(const void *, size_t, int); +void feroceon_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns feroceon_cache_fns __initconst = { + .flush_icache_all = feroceon_flush_icache_all, + .flush_kern_all = feroceon_flush_kern_cache_all, + .flush_kern_louis = feroceon_flush_kern_cache_all, + .flush_user_all = feroceon_flush_user_cache_all, + .flush_user_range = feroceon_flush_user_cache_range, + .coherent_kern_range = feroceon_coherent_kern_range, + .coherent_user_range = feroceon_coherent_user_range, + .flush_kern_dcache_area = feroceon_flush_kern_dcache_area, + .dma_map_area = feroceon_dma_map_area, + .dma_unmap_area = feroceon_dma_unmap_area, + .dma_flush_range = feroceon_dma_flush_range, +}; + +void feroceon_range_flush_kern_dcache_area(void *, size_t); +void feroceon_range_dma_map_area(const void *, size_t, int); +void feroceon_range_dma_flush_range(const void *, const void *); + +struct cpu_cache_fns feroceon_range_cache_fns __initconst = { + .flush_icache_all = feroceon_flush_icache_all, + .flush_kern_all = feroceon_flush_kern_cache_all, + .flush_kern_louis = feroceon_flush_kern_cache_all, + .flush_user_all = feroceon_flush_user_cache_all, + .flush_user_range = feroceon_flush_user_cache_range, + .coherent_kern_range = feroceon_coherent_kern_range, + .coherent_user_range = feroceon_coherent_user_range, + .flush_kern_dcache_area = feroceon_range_flush_kern_dcache_area, + .dma_map_area = feroceon_range_dma_map_area, + .dma_unmap_area = feroceon_dma_unmap_area, + .dma_flush_range = feroceon_range_dma_flush_range, +}; +#endif diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c index b7525b433f3e..4204ffa2d104 100644 --- a/arch/arm/mm/context.c +++ b/arch/arm/mm/context.c @@ -109,7 +109,7 @@ static int contextidr_notifier(struct notifier_block *unused, unsigned long cmd, if (cmd != THREAD_NOTIFY_SWITCH) return NOTIFY_DONE; - pid = task_pid_nr(thread->task) << ASID_BITS; + pid = task_pid_nr(thread_task(thread)) << ASID_BITS; asm volatile( " mrc p15, 0, %0, c13, c0, 1\n" " and %0, %0, %2\n" @@ -240,8 +240,7 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk) unsigned int cpu = smp_processor_id(); u64 asid; - if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq)) - __check_vmalloc_seq(mm); + check_vmalloc_seq(mm); /* * We cannot update the pgd and the ASID atomicly with classic diff --git a/arch/arm/mm/copypage-feroceon.c b/arch/arm/mm/copypage-feroceon.c index 064b19e63571..5fc8ef1e665f 100644 --- a/arch/arm/mm/copypage-feroceon.c +++ b/arch/arm/mm/copypage-feroceon.c @@ -15,6 +15,7 @@ static void feroceon_copy_user_page(void *kto, const void *kfrom) int tmp; asm volatile ("\ +.arch armv5te \n\ 1: ldmia %1!, {r2 - r7, ip, lr} \n\ pld [%1, #0] \n\ pld [%1, #32] \n\ diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c index a94bd08fdec2..7ddd82b9fe8b 100644 --- a/arch/arm/mm/copypage-v4mc.c +++ b/arch/arm/mm/copypage-v4mc.c @@ -13,8 +13,8 @@ #include <linux/init.h> #include <linux/mm.h> #include <linux/highmem.h> +#include <linux/pagemap.h> -#include <asm/pgtable.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> @@ -64,10 +64,11 @@ static void mc_copy_user_page(void *from, void *to) void v4_mc_copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma) { + struct folio *src = page_folio(from); void *kto = kmap_atomic(to); - if (!test_and_set_bit(PG_dcache_clean, &from->flags)) - __flush_dcache_page(page_mapping_file(from), from); + if (!test_and_set_bit(PG_dcache_clean, &src->flags)) + __flush_dcache_folio(folio_flush_mapping(src), src); raw_spin_lock(&minicache_lock); diff --git a/arch/arm/mm/copypage-v6.c b/arch/arm/mm/copypage-v6.c index a6488bb6cfa9..a1a71f36d850 100644 --- a/arch/arm/mm/copypage-v6.c +++ b/arch/arm/mm/copypage-v6.c @@ -8,8 +8,8 @@ #include <linux/spinlock.h> #include <linux/mm.h> #include <linux/highmem.h> +#include <linux/pagemap.h> -#include <asm/pgtable.h> #include <asm/shmparam.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> @@ -69,11 +69,12 @@ static void discard_old_kernel_data(void *kto) static void v6_copy_user_highpage_aliasing(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma) { + struct folio *src = page_folio(from); unsigned int offset = CACHE_COLOUR(vaddr); unsigned long kfrom, kto; - if (!test_and_set_bit(PG_dcache_clean, &from->flags)) - __flush_dcache_page(page_mapping_file(from), from); + if (!test_and_set_bit(PG_dcache_clean, &src->flags)) + __flush_dcache_folio(folio_flush_mapping(src), src); /* FIXME: not highmem safe */ discard_old_kernel_data(page_address(to)); diff --git a/arch/arm/mm/copypage-xsc3.c b/arch/arm/mm/copypage-xsc3.c index 6f0909dda2f9..c86e79677ff9 100644 --- a/arch/arm/mm/copypage-xsc3.c +++ b/arch/arm/mm/copypage-xsc3.c @@ -29,6 +29,7 @@ static void xsc3_mc_copy_user_page(void *kto, const void *kfrom) int tmp; asm volatile ("\ +.arch xscale \n\ pld [%1, #0] \n\ pld [%1, #32] \n\ 1: pld [%1, #64] \n\ @@ -80,6 +81,7 @@ void xsc3_mc_clear_user_highpage(struct page *page, unsigned long vaddr) { void *ptr, *kaddr = kmap_atomic(page); asm volatile ("\ +.arch xscale \n\ mov r1, %2 \n\ mov r2, #0 \n\ mov r3, #0 \n\ diff --git a/arch/arm/mm/copypage-xscale.c b/arch/arm/mm/copypage-xscale.c index 382e1c2855e8..f1e29d3e8193 100644 --- a/arch/arm/mm/copypage-xscale.c +++ b/arch/arm/mm/copypage-xscale.c @@ -13,8 +13,8 @@ #include <linux/init.h> #include <linux/mm.h> #include <linux/highmem.h> +#include <linux/pagemap.h> -#include <asm/pgtable.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> @@ -84,10 +84,11 @@ static void mc_copy_user_page(void *from, void *to) void xscale_mc_copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma) { + struct folio *src = page_folio(from); void *kto = kmap_atomic(to); - if (!test_and_set_bit(PG_dcache_clean, &from->flags)) - __flush_dcache_page(page_mapping_file(from), from); + if (!test_and_set_bit(PG_dcache_clean, &src->flags)) + __flush_dcache_folio(folio_flush_mapping(src), src); raw_spin_lock(&minicache_lock); diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c index 287ef898a55e..fecac107fd0d 100644 --- a/arch/arm/mm/dma-mapping-nommu.c +++ b/arch/arm/mm/dma-mapping-nommu.c @@ -5,11 +5,7 @@ * Copyright (C) 2000-2004 Russell King */ -#include <linux/export.h> -#include <linux/mm.h> -#include <linux/dma-direct.h> -#include <linux/scatterlist.h> - +#include <linux/dma-map-ops.h> #include <asm/cachetype.h> #include <asm/cacheflush.h> #include <asm/outercache.h> @@ -17,65 +13,8 @@ #include "dma.h" -/* - * The generic direct mapping code is used if - * - MMU/MPU is off - * - cpu is v7m w/o cache support - * - device is coherent - * otherwise arm_nommu_dma_ops is used. - * - * arm_nommu_dma_ops rely on consistent DMA memory (please, refer to - * [1] on how to declare such memory). - * - * [1] Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt - */ - -static void *arm_nommu_dma_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, - unsigned long attrs) - -{ - void *ret = dma_alloc_from_global_coherent(dev, size, dma_handle); - - /* - * dma_alloc_from_global_coherent() may fail because: - * - * - no consistent DMA region has been defined, so we can't - * continue. - * - there is no space left in consistent DMA region, so we - * only can fallback to generic allocator if we are - * advertised that consistency is not required. - */ - - WARN_ON_ONCE(ret == NULL); - return ret; -} - -static void arm_nommu_dma_free(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_addr, - unsigned long attrs) -{ - int ret = dma_release_from_global_coherent(get_order(size), cpu_addr); - - WARN_ON_ONCE(ret == 0); -} - -static int arm_nommu_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs) -{ - int ret; - - if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret)) - return ret; - if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) - return ret; - return -ENXIO; -} - - -static void __dma_page_cpu_to_dev(phys_addr_t paddr, size_t size, - enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { dmac_map_area(__va(paddr), size, dir); @@ -85,8 +24,8 @@ static void __dma_page_cpu_to_dev(phys_addr_t paddr, size_t size, outer_clean_range(paddr, paddr + size); } -static void __dma_page_dev_to_cpu(phys_addr_t paddr, size_t size, - enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { if (dir != DMA_TO_DEVICE) { outer_inv_range(paddr, paddr + size); @@ -94,118 +33,20 @@ static void __dma_page_dev_to_cpu(phys_addr_t paddr, size_t size, } } -static dma_addr_t arm_nommu_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, - unsigned long attrs) -{ - dma_addr_t handle = page_to_phys(page) + offset; - - __dma_page_cpu_to_dev(handle, size, dir); - - return handle; -} - -static void arm_nommu_dma_unmap_page(struct device *dev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - __dma_page_dev_to_cpu(handle, size, dir); -} - - -static int arm_nommu_dma_map_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, - unsigned long attrs) -{ - int i; - struct scatterlist *sg; - - for_each_sg(sgl, sg, nents, i) { - sg_dma_address(sg) = sg_phys(sg); - sg_dma_len(sg) = sg->length; - __dma_page_cpu_to_dev(sg_dma_address(sg), sg_dma_len(sg), dir); - } - - return nents; -} - -static void arm_nommu_dma_unmap_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, - unsigned long attrs) -{ - struct scatterlist *sg; - int i; - - for_each_sg(sgl, sg, nents, i) - __dma_page_dev_to_cpu(sg_dma_address(sg), sg_dma_len(sg), dir); -} - -static void arm_nommu_dma_sync_single_for_device(struct device *dev, - dma_addr_t handle, size_t size, enum dma_data_direction dir) -{ - __dma_page_cpu_to_dev(handle, size, dir); -} - -static void arm_nommu_dma_sync_single_for_cpu(struct device *dev, - dma_addr_t handle, size_t size, enum dma_data_direction dir) -{ - __dma_page_cpu_to_dev(handle, size, dir); -} - -static void arm_nommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir) -{ - struct scatterlist *sg; - int i; - - for_each_sg(sgl, sg, nents, i) - __dma_page_cpu_to_dev(sg_dma_address(sg), sg_dma_len(sg), dir); -} - -static void arm_nommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir) -{ - struct scatterlist *sg; - int i; - - for_each_sg(sgl, sg, nents, i) - __dma_page_dev_to_cpu(sg_dma_address(sg), sg_dma_len(sg), dir); -} - -const struct dma_map_ops arm_nommu_dma_ops = { - .alloc = arm_nommu_dma_alloc, - .free = arm_nommu_dma_free, - .mmap = arm_nommu_dma_mmap, - .map_page = arm_nommu_dma_map_page, - .unmap_page = arm_nommu_dma_unmap_page, - .map_sg = arm_nommu_dma_map_sg, - .unmap_sg = arm_nommu_dma_unmap_sg, - .sync_single_for_device = arm_nommu_dma_sync_single_for_device, - .sync_single_for_cpu = arm_nommu_dma_sync_single_for_cpu, - .sync_sg_for_device = arm_nommu_dma_sync_sg_for_device, - .sync_sg_for_cpu = arm_nommu_dma_sync_sg_for_cpu, -}; -EXPORT_SYMBOL(arm_nommu_dma_ops); - -void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - const struct iommu_ops *iommu, bool coherent) +void arch_setup_dma_ops(struct device *dev, bool coherent) { if (IS_ENABLED(CONFIG_CPU_V7M)) { /* * Cache support for v7m is optional, so can be treated as * coherent if no cache has been detected. Note that it is not - * enough to check if MPU is in use or not since in absense of + * enough to check if MPU is in use or not since in absence of * MPU system memory map is used. */ - dev->archdata.dma_coherent = (cacheid) ? coherent : true; + dev->dma_coherent = cacheid ? coherent : true; } else { /* * Assume coherent DMA in case MMU/MPU has not been set up. */ - dev->archdata.dma_coherent = (get_cr() & CR_M) ? coherent : true; + dev->dma_coherent = (get_cr() & CR_M) ? coherent : true; } - - if (!dev->archdata.dma_coherent) - set_dma_ops(dev, &arm_nommu_dma_ops); } diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index e822af0d9219..88c2d68a69c9 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -15,9 +15,7 @@ #include <linux/init.h> #include <linux/device.h> #include <linux/dma-direct.h> -#include <linux/dma-mapping.h> -#include <linux/dma-noncoherent.h> -#include <linux/dma-contiguous.h> +#include <linux/dma-map-ops.h> #include <linux/highmem.h> #include <linux/memblock.h> #include <linux/slab.h> @@ -27,7 +25,7 @@ #include <linux/sizes.h> #include <linux/cma.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/highmem.h> #include <asm/cacheflush.h> #include <asm/tlbflush.h> @@ -35,8 +33,7 @@ #include <asm/dma-iommu.h> #include <asm/mach/map.h> #include <asm/system_info.h> -#include <asm/dma-contiguous.h> -#include <xen/swiotlb-xen.h> +#include <asm/xen/xen-ops.h> #include "dma.h" #include "mm.h" @@ -106,161 +103,6 @@ static struct arm_dma_buffer *arm_dma_buffer_find(void *virt) * before transfers and delay cache invalidation until transfer completion. * */ -static void __dma_page_cpu_to_dev(struct page *, unsigned long, - size_t, enum dma_data_direction); -static void __dma_page_dev_to_cpu(struct page *, unsigned long, - size_t, enum dma_data_direction); - -/** - * arm_dma_map_page - map a portion of a page for streaming DMA - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @page: page that buffer resides in - * @offset: offset into page for start of buffer - * @size: size of buffer to map - * @dir: DMA transfer direction - * - * Ensure that any data held in the cache is appropriately discarded - * or written back. - * - * The device owns this memory once this call has completed. The CPU - * can regain ownership by calling dma_unmap_page(). - */ -static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) - __dma_page_cpu_to_dev(page, offset, size, dir); - return pfn_to_dma(dev, page_to_pfn(page)) + offset; -} - -static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - return pfn_to_dma(dev, page_to_pfn(page)) + offset; -} - -/** - * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page() - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @handle: DMA address of buffer - * @size: size of buffer (same as passed to dma_map_page) - * @dir: DMA transfer direction (same as passed to dma_map_page) - * - * Unmap a page streaming mode DMA translation. The handle and size - * must match what was provided in the previous dma_map_page() call. - * All other usages are undefined. - * - * After this call, reads by the CPU to the buffer are guaranteed to see - * whatever the device wrote there. - */ -static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, unsigned long attrs) -{ - if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) - __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), - handle & ~PAGE_MASK, size, dir); -} - -static void arm_dma_sync_single_for_cpu(struct device *dev, - dma_addr_t handle, size_t size, enum dma_data_direction dir) -{ - unsigned int offset = handle & (PAGE_SIZE - 1); - struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); - __dma_page_dev_to_cpu(page, offset, size, dir); -} - -static void arm_dma_sync_single_for_device(struct device *dev, - dma_addr_t handle, size_t size, enum dma_data_direction dir) -{ - unsigned int offset = handle & (PAGE_SIZE - 1); - struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); - __dma_page_cpu_to_dev(page, offset, size, dir); -} - -const struct dma_map_ops arm_dma_ops = { - .alloc = arm_dma_alloc, - .free = arm_dma_free, - .mmap = arm_dma_mmap, - .get_sgtable = arm_dma_get_sgtable, - .map_page = arm_dma_map_page, - .unmap_page = arm_dma_unmap_page, - .map_sg = arm_dma_map_sg, - .unmap_sg = arm_dma_unmap_sg, - .map_resource = dma_direct_map_resource, - .sync_single_for_cpu = arm_dma_sync_single_for_cpu, - .sync_single_for_device = arm_dma_sync_single_for_device, - .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu, - .sync_sg_for_device = arm_dma_sync_sg_for_device, - .dma_supported = arm_dma_supported, - .get_required_mask = dma_direct_get_required_mask, -}; -EXPORT_SYMBOL(arm_dma_ops); - -static void *arm_coherent_dma_alloc(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, unsigned long attrs); -static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, unsigned long attrs); -static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs); - -const struct dma_map_ops arm_coherent_dma_ops = { - .alloc = arm_coherent_dma_alloc, - .free = arm_coherent_dma_free, - .mmap = arm_coherent_dma_mmap, - .get_sgtable = arm_dma_get_sgtable, - .map_page = arm_coherent_dma_map_page, - .map_sg = arm_dma_map_sg, - .map_resource = dma_direct_map_resource, - .dma_supported = arm_dma_supported, - .get_required_mask = dma_direct_get_required_mask, -}; -EXPORT_SYMBOL(arm_coherent_dma_ops); - -static int __dma_supported(struct device *dev, u64 mask, bool warn) -{ - unsigned long max_dma_pfn = min(max_pfn, arm_dma_pfn_limit); - - /* - * Translate the device's DMA mask to a PFN limit. This - * PFN number includes the page which we can DMA to. - */ - if (dma_to_pfn(dev, mask) < max_dma_pfn) { - if (warn) - dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n", - mask, - dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1, - max_dma_pfn + 1); - return 0; - } - - return 1; -} - -static u64 get_coherent_dma_mask(struct device *dev) -{ - u64 mask = (u64)DMA_BIT_MASK(32); - - if (dev) { - mask = dev->coherent_dma_mask; - - /* - * Sanity check the DMA mask - it must be non-zero, and - * must be able to be satisfied by a DMA allocation. - */ - if (mask == 0) { - dev_warn(dev, "coherent DMA mask is unset\n"); - return 0; - } - - if (!__dma_supported(dev, mask, true)) - return 0; - } - - return mask; -} static void __dma_clear_buffer(struct page *page, size_t size, int coherent_flag) { @@ -406,6 +248,7 @@ out: */ postcore_initcall(atomic_pool_init); +#ifdef CONFIG_CMA_AREAS struct dma_contig_early_reserve { phys_addr_t base; unsigned long size; @@ -415,12 +258,14 @@ static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata; static int dma_mmu_remap_num __initdata; +#ifdef CONFIG_DMA_CMA void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) { dma_mmu_remap[dma_mmu_remap_num].base = base; dma_mmu_remap[dma_mmu_remap_num].size = size; dma_mmu_remap_num++; } +#endif void __init dma_contiguous_remap(void) { @@ -460,10 +305,11 @@ void __init dma_contiguous_remap(void) iotable_init(&map, 1); } } +#endif static int __dma_update_pte(pte_t *pte, unsigned long addr, void *data) { - struct page *page = virt_to_page(addr); + struct page *page = virt_to_page((void *)addr); pgprot_t prot = *(pgprot_t *)data; set_pte_ext(pte, mk_pte(page, prot), 0); @@ -688,7 +534,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, pgprot_t prot, bool is_coherent, unsigned long attrs, const void *caller) { - u64 mask = get_coherent_dma_mask(dev); + u64 mask = min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); struct page *page = NULL; void *addr; bool allowblock, cma; @@ -712,9 +558,6 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, } #endif - if (!mask) - return NULL; - buf = kzalloc(sizeof(*buf), gfp & ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM)); if (!buf) @@ -723,19 +566,11 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, if (mask < 0xffffffffULL) gfp |= GFP_DMA; - /* - * Following is a work-around (a.k.a. hack) to prevent pages - * with __GFP_COMP being passed to split_page() which cannot - * handle them. The real problem is that this flag probably - * should be 0 on ARM as it is not supported on this - * platform; see CONFIG_HUGETLBFS. - */ - gfp &= ~(__GFP_COMP); args.gfp = gfp; *handle = DMA_MAPPING_ERROR; allowblock = gfpflags_allow_blocking(gfp); - cma = allowblock ? dev_get_cma_area(dev) : false; + cma = allowblock ? dev_get_cma_area(dev) : NULL; if (cma) buf->allocator = &cma_allocator; @@ -751,7 +586,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, if (page) { unsigned long flags; - *handle = pfn_to_dma(dev, page_to_pfn(page)); + *handle = phys_to_dma(dev, page_to_phys(page)); buf->virt = args.want_vaddr ? addr : page; spin_lock_irqsave(&arm_dma_bufs_lock, flags); @@ -765,74 +600,13 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, } /* - * Allocate DMA-coherent memory space and return both the kernel remapped - * virtual and bus address for that space. - */ -void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, - gfp_t gfp, unsigned long attrs) -{ - pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); - - return __dma_alloc(dev, size, handle, gfp, prot, false, - attrs, __builtin_return_address(0)); -} - -static void *arm_coherent_dma_alloc(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, unsigned long attrs) -{ - return __dma_alloc(dev, size, handle, gfp, PAGE_KERNEL, true, - attrs, __builtin_return_address(0)); -} - -static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs) -{ - int ret = -ENXIO; - unsigned long nr_vma_pages = vma_pages(vma); - unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; - unsigned long pfn = dma_to_pfn(dev, dma_addr); - unsigned long off = vma->vm_pgoff; - - if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) - return ret; - - if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) { - ret = remap_pfn_range(vma, vma->vm_start, - pfn + off, - vma->vm_end - vma->vm_start, - vma->vm_page_prot); - } - - return ret; -} - -/* - * Create userspace mapping for the DMA-coherent memory. - */ -static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs) -{ - return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); -} - -int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs) -{ - vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); - return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); -} - -/* * Free a buffer as defined by the above mapping. */ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle, unsigned long attrs, bool is_coherent) { - struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); + struct page *page = phys_to_page(dma_to_phys(dev, handle)); struct arm_dma_buffer *buf; struct arm_dma_free_args args = { .dev = dev, @@ -850,40 +624,6 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, kfree(buf); } -void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, unsigned long attrs) -{ - __arm_dma_free(dev, size, cpu_addr, handle, attrs, false); -} - -static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, unsigned long attrs) -{ - __arm_dma_free(dev, size, cpu_addr, handle, attrs, true); -} - -int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, - void *cpu_addr, dma_addr_t handle, size_t size, - unsigned long attrs) -{ - unsigned long pfn = dma_to_pfn(dev, handle); - struct page *page; - int ret; - - /* If the PFN is not valid, we do not have a struct page */ - if (!pfn_valid(pfn)) - return -ENXIO; - - page = pfn_to_page(pfn); - - ret = sg_alloc_table(sgt, 1, GFP_KERNEL); - if (unlikely(ret)) - return ret; - - sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); - return 0; -} - static void dma_cache_maint_page(struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, void (*op)(const void *, size_t, int)) @@ -933,8 +673,7 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset, /* * Make an area consistent for devices. - * Note: Drivers should NOT use this function directly, as it will break - * platforms with CONFIG_DMABOUNCE. + * Note: Drivers should NOT use this function directly. * Use the driver DMA support - see dma-mapping.h (dma_sync_*) */ static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, @@ -970,146 +709,23 @@ static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, * Mark the D-cache clean for these pages to avoid extra flushing. */ if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) { - unsigned long pfn; - size_t left = size; - - pfn = page_to_pfn(page) + off / PAGE_SIZE; - off %= PAGE_SIZE; - if (off) { - pfn++; - left -= PAGE_SIZE - off; - } - while (left >= PAGE_SIZE) { - page = pfn_to_page(pfn++); - set_bit(PG_dcache_clean, &page->flags); - left -= PAGE_SIZE; - } - } -} + struct folio *folio = pfn_folio(paddr / PAGE_SIZE); + size_t offset = offset_in_folio(folio, paddr); -/** - * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @sg: list of buffers - * @nents: number of buffers to map - * @dir: DMA transfer direction - * - * Map a set of buffers described by scatterlist in streaming mode for DMA. - * This is the scatter-gather version of the dma_map_single interface. - * Here the scatter gather list elements are each tagged with the - * appropriate dma address and length. They are obtained via - * sg_dma_{address,length}. - * - * Device ownership issues as mentioned for dma_map_single are the same - * here. - */ -int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, unsigned long attrs) -{ - const struct dma_map_ops *ops = get_dma_ops(dev); - struct scatterlist *s; - int i, j; + for (;;) { + size_t sz = folio_size(folio) - offset; - for_each_sg(sg, s, nents, i) { -#ifdef CONFIG_NEED_SG_DMA_LENGTH - s->dma_length = s->length; -#endif - s->dma_address = ops->map_page(dev, sg_page(s), s->offset, - s->length, dir, attrs); - if (dma_mapping_error(dev, s->dma_address)) - goto bad_mapping; + if (size < sz) + break; + if (!offset) + set_bit(PG_dcache_clean, &folio->flags); + offset = 0; + size -= sz; + if (!size) + break; + folio = folio_next(folio); + } } - return nents; - - bad_mapping: - for_each_sg(sg, s, i, j) - ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); - return 0; -} - -/** - * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @sg: list of buffers - * @nents: number of buffers to unmap (same as was passed to dma_map_sg) - * @dir: DMA transfer direction (same as was passed to dma_map_sg) - * - * Unmap a set of streaming mode DMA translations. Again, CPU access - * rules concerning calls here are the same as for dma_unmap_single(). - */ -void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, unsigned long attrs) -{ - const struct dma_map_ops *ops = get_dma_ops(dev); - struct scatterlist *s; - - int i; - - for_each_sg(sg, s, nents, i) - ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); -} - -/** - * arm_dma_sync_sg_for_cpu - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @sg: list of buffers - * @nents: number of buffers to map (returned from dma_map_sg) - * @dir: DMA transfer direction (same as was passed to dma_map_sg) - */ -void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir) -{ - const struct dma_map_ops *ops = get_dma_ops(dev); - struct scatterlist *s; - int i; - - for_each_sg(sg, s, nents, i) - ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length, - dir); -} - -/** - * arm_dma_sync_sg_for_device - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @sg: list of buffers - * @nents: number of buffers to map (returned from dma_map_sg) - * @dir: DMA transfer direction (same as was passed to dma_map_sg) - */ -void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir) -{ - const struct dma_map_ops *ops = get_dma_ops(dev); - struct scatterlist *s; - int i; - - for_each_sg(sg, s, nents, i) - ops->sync_single_for_device(dev, sg_dma_address(s), s->length, - dir); -} - -/* - * Return whether the given device DMA address mask can be supported - * properly. For example, if your device can only drive the low 24-bits - * during bus mastering, then you would pass 0x00ffffff as the mask - * to this function. - */ -int arm_dma_supported(struct device *dev, u64 mask) -{ - return __dma_supported(dev, mask, false); -} - -static const struct dma_map_ops *arm_get_dma_map_ops(bool coherent) -{ - /* - * When CONFIG_ARM_LPAE is set, physical address can extend above - * 32-bits, which then can't be addressed by devices that only support - * 32-bit DMA. - * Use the generic dma-direct / swiotlb ops code in that case, as that - * handles bounce buffering for us. - */ - if (IS_ENABLED(CONFIG_ARM_LPAE)) - return NULL; - return coherent ? &arm_coherent_dma_ops : &arm_dma_ops; } #ifdef CONFIG_ARM_DMA_USE_IOMMU @@ -1243,10 +859,7 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, int i = 0; int order_idx = 0; - if (array_size <= PAGE_SIZE) - pages = kzalloc(array_size, GFP_KERNEL); - else - pages = vzalloc(array_size); + pages = kvzalloc(array_size, GFP_KERNEL); if (!pages) return NULL; @@ -1372,7 +985,8 @@ __iommu_create_mapping(struct device *dev, struct page **pages, size_t size, len = (j - i) << PAGE_SHIFT; ret = iommu_map(mapping->domain, iova, phys, len, - __dma_info_to_prot(DMA_BIDIRECTIONAL, attrs)); + __dma_info_to_prot(DMA_BIDIRECTIONAL, attrs), + GFP_KERNEL); if (ret < 0) goto fail; iova += len; @@ -1458,13 +1072,13 @@ static void __iommu_free_atomic(struct device *dev, void *cpu_addr, __free_from_pool(cpu_addr, size); } -static void *__arm_iommu_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, unsigned long attrs, - int coherent_flag) +static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp, unsigned long attrs) { pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); struct page **pages; void *addr = NULL; + int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL; *handle = DMA_MAPPING_ERROR; size = PAGE_ALIGN(size); @@ -1473,15 +1087,6 @@ static void *__arm_iommu_alloc_attrs(struct device *dev, size_t size, return __iommu_alloc_simple(dev, size, gfp, handle, coherent_flag, attrs); - /* - * Following is a work-around (a.k.a. hack) to prevent pages - * with __GFP_COMP being passed to split_page() which cannot - * handle them. The real problem is that this flag probably - * should be 0 on ARM as it is not supported on this - * platform; see CONFIG_HUGETLBFS. - */ - gfp &= ~(__GFP_COMP); - pages = __iommu_alloc_buffer(dev, size, gfp, attrs, coherent_flag); if (!pages) return NULL; @@ -1507,19 +1112,7 @@ err_buffer: return NULL; } -static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, unsigned long attrs) -{ - return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, NORMAL); -} - -static void *arm_coherent_iommu_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, unsigned long attrs) -{ - return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, COHERENT); -} - -static int __arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, +static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs) { @@ -1533,35 +1126,24 @@ static int __arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma if (vma->vm_pgoff >= nr_pages) return -ENXIO; + if (!dev->dma_coherent) + vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); + err = vm_map_pages(vma, pages, nr_pages); if (err) pr_err("Remapping memory failed: %d\n", err); return err; } -static int arm_iommu_mmap_attrs(struct device *dev, - struct vm_area_struct *vma, void *cpu_addr, - dma_addr_t dma_addr, size_t size, unsigned long attrs) -{ - vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); - - return __arm_iommu_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, attrs); -} - -static int arm_coherent_iommu_mmap_attrs(struct device *dev, - struct vm_area_struct *vma, void *cpu_addr, - dma_addr_t dma_addr, size_t size, unsigned long attrs) -{ - return __arm_iommu_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, attrs); -} /* * free a page as defined by the above mapping. * Must not be called with IRQs disabled. */ -static void __arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, unsigned long attrs, int coherent_flag) +static void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle, unsigned long attrs) { + int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL; struct page **pages; size = PAGE_ALIGN(size); @@ -1583,19 +1165,6 @@ static void __arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_ad __iommu_free_buffer(dev, pages, size, attrs); } -static void arm_iommu_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t handle, - unsigned long attrs) -{ - __arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, NORMAL); -} - -static void arm_coherent_iommu_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t handle, unsigned long attrs) -{ - __arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, COHERENT); -} - static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs) @@ -1615,8 +1184,7 @@ static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, */ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, size_t size, dma_addr_t *handle, - enum dma_data_direction dir, unsigned long attrs, - bool is_coherent) + enum dma_data_direction dir, unsigned long attrs) { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova, iova_base; @@ -1636,12 +1204,13 @@ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, phys_addr_t phys = page_to_phys(sg_page(s)); unsigned int len = PAGE_ALIGN(s->offset + s->length); - if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); prot = __dma_info_to_prot(dir, attrs); - ret = iommu_map(mapping->domain, iova, phys, len, prot); + ret = iommu_map(mapping->domain, iova, phys, len, prot, + GFP_KERNEL); if (ret < 0) goto fail; count += len >> PAGE_SHIFT; @@ -1656,12 +1225,23 @@ fail: return ret; } -static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, unsigned long attrs, - bool is_coherent) +/** + * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA + * @dev: valid struct device pointer + * @sg: list of buffers + * @nents: number of buffers to map + * @dir: DMA transfer direction + * + * Map a set of buffers described by scatterlist in streaming mode for DMA. + * The scatter gather list elements are merged together (if possible) and + * tagged with the appropriate dma address and length. They are obtained via + * sg_dma_{address,length}. + */ +static int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, unsigned long attrs) { struct scatterlist *s = sg, *dma = sg, *start = sg; - int i, count = 0; + int i, count = 0, ret; unsigned int offset = s->offset; unsigned int size = s->offset + s->length; unsigned int max = dma_get_max_seg_size(dev); @@ -1669,12 +1249,12 @@ static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, for (i = 1; i < nents; i++) { s = sg_next(s); - s->dma_address = DMA_MAPPING_ERROR; s->dma_length = 0; if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { - if (__map_sg_chunk(dev, start, size, &dma->dma_address, - dir, attrs, is_coherent) < 0) + ret = __map_sg_chunk(dev, start, size, + &dma->dma_address, dir, attrs); + if (ret < 0) goto bad_mapping; dma->dma_address += offset; @@ -1687,8 +1267,8 @@ static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, } size += s->length; } - if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs, - is_coherent) < 0) + ret = __map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs); + if (ret < 0) goto bad_mapping; dma->dma_address += offset; @@ -1699,48 +1279,25 @@ static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, bad_mapping: for_each_sg(sg, s, count, i) __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); - return 0; -} - -/** - * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA - * @dev: valid struct device pointer - * @sg: list of buffers - * @nents: number of buffers to map - * @dir: DMA transfer direction - * - * Map a set of i/o coherent buffers described by scatterlist in streaming - * mode for DMA. The scatter gather list elements are merged together (if - * possible) and tagged with the appropriate dma address and length. They are - * obtained via sg_dma_{address,length}. - */ -static int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - return __iommu_map_sg(dev, sg, nents, dir, attrs, true); + if (ret == -ENOMEM) + return ret; + return -EINVAL; } /** - * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA + * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg * @dev: valid struct device pointer * @sg: list of buffers - * @nents: number of buffers to map - * @dir: DMA transfer direction + * @nents: number of buffers to unmap (same as was passed to dma_map_sg) + * @dir: DMA transfer direction (same as was passed to dma_map_sg) * - * Map a set of buffers described by scatterlist in streaming mode for DMA. - * The scatter gather list elements are merged together (if possible) and - * tagged with the appropriate dma address and length. They are obtained via - * sg_dma_{address,length}. + * Unmap a set of streaming mode DMA translations. Again, CPU access + * rules concerning calls here are the same as for dma_unmap_single(). */ -static int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - return __iommu_map_sg(dev, sg, nents, dir, attrs, false); -} - -static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, - unsigned long attrs, bool is_coherent) +static void arm_iommu_unmap_sg(struct device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction dir, + unsigned long attrs) { struct scatterlist *s; int i; @@ -1749,48 +1306,13 @@ static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, if (sg_dma_len(s)) __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); - if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); } } /** - * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg - * @dev: valid struct device pointer - * @sg: list of buffers - * @nents: number of buffers to unmap (same as was passed to dma_map_sg) - * @dir: DMA transfer direction (same as was passed to dma_map_sg) - * - * Unmap a set of streaming mode DMA translations. Again, CPU access - * rules concerning calls here are the same as for dma_unmap_single(). - */ -static void arm_coherent_iommu_unmap_sg(struct device *dev, - struct scatterlist *sg, int nents, enum dma_data_direction dir, - unsigned long attrs) -{ - __iommu_unmap_sg(dev, sg, nents, dir, attrs, true); -} - -/** - * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg - * @dev: valid struct device pointer - * @sg: list of buffers - * @nents: number of buffers to unmap (same as was passed to dma_map_sg) - * @dir: DMA transfer direction (same as was passed to dma_map_sg) - * - * Unmap a set of streaming mode DMA translations. Again, CPU access - * rules concerning calls here are the same as for dma_unmap_single(). - */ -static void arm_iommu_unmap_sg(struct device *dev, - struct scatterlist *sg, int nents, - enum dma_data_direction dir, - unsigned long attrs) -{ - __iommu_unmap_sg(dev, sg, nents, dir, attrs, false); -} - -/** * arm_iommu_sync_sg_for_cpu * @dev: valid struct device pointer * @sg: list of buffers @@ -1804,6 +1326,9 @@ static void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *s; int i; + if (dev->dma_coherent) + return; + for_each_sg(sg, s, nents, i) __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); @@ -1823,22 +1348,24 @@ static void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *s; int i; + if (dev->dma_coherent) + return; + for_each_sg(sg, s, nents, i) __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); } - /** - * arm_coherent_iommu_map_page + * arm_iommu_map_page * @dev: valid struct device pointer * @page: page that buffer resides in * @offset: offset into page for start of buffer * @size: size of buffer to map * @dir: DMA transfer direction * - * Coherent IOMMU aware version of arm_dma_map_page() + * IOMMU aware version of arm_dma_map_page() */ -static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page, +static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs) { @@ -1846,13 +1373,17 @@ static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *p dma_addr_t dma_addr; int ret, prot, len = PAGE_ALIGN(size + offset); + if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + __dma_page_cpu_to_dev(page, offset, size, dir); + dma_addr = __alloc_iova(mapping, len); if (dma_addr == DMA_MAPPING_ERROR) return dma_addr; prot = __dma_info_to_prot(dir, attrs); - ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot); + ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, + prot, GFP_KERNEL); if (ret < 0) goto fail; @@ -1863,50 +1394,6 @@ fail: } /** - * arm_iommu_map_page - * @dev: valid struct device pointer - * @page: page that buffer resides in - * @offset: offset into page for start of buffer - * @size: size of buffer to map - * @dir: DMA transfer direction - * - * IOMMU aware version of arm_dma_map_page() - */ -static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) - __dma_page_cpu_to_dev(page, offset, size, dir); - - return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs); -} - -/** - * arm_coherent_iommu_unmap_page - * @dev: valid struct device pointer - * @handle: DMA address of buffer - * @size: size of buffer (same as passed to dma_map_page) - * @dir: DMA transfer direction (same as passed to dma_map_page) - * - * Coherent IOMMU aware version of arm_dma_unmap_page() - */ -static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, unsigned long attrs) -{ - struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); - dma_addr_t iova = handle & PAGE_MASK; - int offset = handle & ~PAGE_MASK; - int len = PAGE_ALIGN(size + offset); - - if (!iova) - return; - - iommu_unmap(mapping->domain, iova, len); - __free_iova(mapping, iova, len); -} - -/** * arm_iommu_unmap_page * @dev: valid struct device pointer * @handle: DMA address of buffer @@ -1920,15 +1407,17 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; - struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); + struct page *page; int offset = handle & ~PAGE_MASK; int len = PAGE_ALIGN(size + offset); if (!iova) return; - if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) { + page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); __dma_page_dev_to_cpu(page, offset, size, dir); + } iommu_unmap(mapping->domain, iova, len); __free_iova(mapping, iova, len); @@ -1958,7 +1447,7 @@ static dma_addr_t arm_iommu_map_resource(struct device *dev, prot = __dma_info_to_prot(dir, attrs) | IOMMU_MMIO; - ret = iommu_map(mapping->domain, dma_addr, addr, len, prot); + ret = iommu_map(mapping->domain, dma_addr, addr, len, prot, GFP_KERNEL); if (ret < 0) goto fail; @@ -1996,12 +1485,13 @@ static void arm_iommu_sync_single_for_cpu(struct device *dev, { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; - struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); + struct page *page; unsigned int offset = handle & ~PAGE_MASK; - if (!iova) + if (dev->dma_coherent || !iova) return; + page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); __dma_page_dev_to_cpu(page, offset, size, dir); } @@ -2010,12 +1500,13 @@ static void arm_iommu_sync_single_for_device(struct device *dev, { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; - struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); + struct page *page; unsigned int offset = handle & ~PAGE_MASK; - if (!iova) + if (dev->dma_coherent || !iova) return; + page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); __dma_page_cpu_to_dev(page, offset, size, dir); } @@ -2037,31 +1528,11 @@ static const struct dma_map_ops iommu_ops = { .map_resource = arm_iommu_map_resource, .unmap_resource = arm_iommu_unmap_resource, - - .dma_supported = arm_dma_supported, -}; - -static const struct dma_map_ops iommu_coherent_ops = { - .alloc = arm_coherent_iommu_alloc_attrs, - .free = arm_coherent_iommu_free_attrs, - .mmap = arm_coherent_iommu_mmap_attrs, - .get_sgtable = arm_iommu_get_sgtable, - - .map_page = arm_coherent_iommu_map_page, - .unmap_page = arm_coherent_iommu_unmap_page, - - .map_sg = arm_coherent_iommu_map_sg, - .unmap_sg = arm_coherent_iommu_unmap_sg, - - .map_resource = arm_iommu_map_resource, - .unmap_resource = arm_iommu_unmap_resource, - - .dma_supported = arm_dma_supported, }; /** * arm_iommu_create_mapping - * @bus: pointer to the bus holding the client device (for IOMMU calls) + * @dev: pointer to the client device (for IOMMU calls) * @base: start address of the valid IO address space * @size: maximum size of the valid IO address space * @@ -2073,7 +1544,7 @@ static const struct dma_map_ops iommu_coherent_ops = { * arm_iommu_attach_device function. */ struct dma_iommu_mapping * -arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size) +arm_iommu_create_mapping(struct device *dev, dma_addr_t base, u64 size) { unsigned int bits = size >> PAGE_SHIFT; unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long); @@ -2114,9 +1585,11 @@ arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size) spin_lock_init(&mapping->lock); - mapping->domain = iommu_domain_alloc(bus); - if (!mapping->domain) + mapping->domain = iommu_paging_domain_alloc(dev); + if (IS_ERR(mapping->domain)) { + err = PTR_ERR(mapping->domain); goto err4; + } kref_init(&mapping->kref); return mapping; @@ -2232,40 +1705,36 @@ void arm_iommu_detach_device(struct device *dev) iommu_detach_device(mapping->domain, dev); kref_put(&mapping->kref, release_iommu_mapping); to_dma_iommu_mapping(dev) = NULL; - set_dma_ops(dev, arm_get_dma_map_ops(dev->archdata.dma_coherent)); + set_dma_ops(dev, NULL); pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); } EXPORT_SYMBOL_GPL(arm_iommu_detach_device); -static const struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent) -{ - return coherent ? &iommu_coherent_ops : &iommu_ops; -} - -static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, - const struct iommu_ops *iommu) +static void arm_setup_iommu_dma_ops(struct device *dev) { struct dma_iommu_mapping *mapping; + u64 dma_base = 0, size = 1ULL << 32; - if (!iommu) - return false; - - mapping = arm_iommu_create_mapping(dev->bus, dma_base, size); + if (dev->dma_range_map) { + dma_base = dma_range_map_min(dev->dma_range_map); + size = dma_range_map_max(dev->dma_range_map) - dma_base; + } + mapping = arm_iommu_create_mapping(dev, dma_base, size); if (IS_ERR(mapping)) { pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n", size, dev_name(dev)); - return false; + return; } if (__arm_iommu_attach_device(dev, mapping)) { pr_warn("Failed to attached device %s to IOMMU_mapping\n", dev_name(dev)); arm_iommu_release_mapping(mapping); - return false; + return; } - return true; + set_dma_ops(dev, &iommu_ops); } static void arm_teardown_iommu_dma_ops(struct device *dev) @@ -2281,27 +1750,24 @@ static void arm_teardown_iommu_dma_ops(struct device *dev) #else -static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, - const struct iommu_ops *iommu) +static void arm_setup_iommu_dma_ops(struct device *dev) { - return false; } static void arm_teardown_iommu_dma_ops(struct device *dev) { } -#define arm_get_iommu_dma_map_ops arm_get_dma_map_ops - #endif /* CONFIG_ARM_DMA_USE_IOMMU */ -void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, - const struct iommu_ops *iommu, bool coherent) +void arch_setup_dma_ops(struct device *dev, bool coherent) { - const struct dma_map_ops *dma_ops; - - dev->archdata.dma_coherent = coherent; -#ifdef CONFIG_SWIOTLB - dev->dma_coherent = coherent; -#endif + /* + * Due to legacy code that sets the ->dma_coherent flag from a bus + * notifier we can't just assign coherent to the ->dma_coherent flag + * here, but instead have to make sure we only set but never clear it + * for now. + */ + if (coherent) + dev->dma_coherent = true; /* * Don't override the dma_ops if they have already been set. Ideally @@ -2311,17 +1777,10 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, if (dev->dma_ops) return; - if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu)) - dma_ops = arm_get_iommu_dma_map_ops(coherent); - else - dma_ops = arm_get_dma_map_ops(coherent); + if (device_iommu_mapped(dev)) + arm_setup_iommu_dma_ops(dev); - set_dma_ops(dev, dma_ops); - -#ifdef CONFIG_XEN - if (xen_initial_domain()) - dev->dma_ops = &xen_swiotlb_dma_ops; -#endif + xen_setup_dma_ops(dev); dev->archdata.dma_ops_setup = true; } @@ -2335,7 +1794,6 @@ void arch_teardown_dma_ops(struct device *dev) set_dma_ops(dev, NULL); } -#ifdef CONFIG_SWIOTLB void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, enum dma_data_direction dir) { @@ -2363,4 +1821,3 @@ void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, { __arm_dma_free(dev, size, cpu_addr, dma_handle, attrs, false); } -#endif /* CONFIG_SWIOTLB */ diff --git a/arch/arm/mm/dump.c b/arch/arm/mm/dump.c index 7d6291f23251..cd032522d902 100644 --- a/arch/arm/mm/dump.c +++ b/arch/arm/mm/dump.c @@ -15,15 +15,18 @@ #include <asm/domain.h> #include <asm/fixmap.h> -#include <asm/memory.h> -#include <asm/pgtable.h> +#include <asm/page.h> #include <asm/ptdump.h> static struct addr_marker address_markers[] = { +#ifdef CONFIG_KASAN + { KASAN_SHADOW_START, "Kasan shadow start"}, + { KASAN_SHADOW_END, "Kasan shadow end"}, +#endif { MODULES_VADDR, "Modules" }, { PAGE_OFFSET, "Kernel Mapping" }, { 0, "vmalloc() Area" }, - { VMALLOC_END, "vmalloc() End" }, + { FDT_FIXED_BASE, "FDT Area" }, { FIXADDR_START, "Fixmap Area" }, { VECTORS_BASE, "Vectors" }, { VECTORS_BASE + PAGE_SIZE * 2, "Vectors End" }, @@ -197,6 +200,7 @@ static const struct prot_bits section_bits[] = { }; struct pg_level { + const char *name; const struct prot_bits *bits; size_t num; u64 mask; @@ -207,11 +211,14 @@ struct pg_level { static struct pg_level pg_level[] = { { }, { /* pgd */ + }, { /* p4d */ }, { /* pud */ }, { /* pmd */ + .name = (CONFIG_PGTABLE_LEVELS > 2) ? "PMD" : "PGD", .bits = section_bits, .num = ARRAY_SIZE(section_bits), }, { /* pte */ + .name = "PTE", .bits = pte_bits, .num = ARRAY_SIZE(pte_bits), }, @@ -278,7 +285,8 @@ static void note_page(struct pg_state *st, unsigned long addr, delta >>= 10; unit++; } - pt_dump_seq_printf(st->seq, "%9lu%c", delta, *unit); + pt_dump_seq_printf(st->seq, "%9lu%c %s", delta, *unit, + pg_level[st->level].name); if (st->current_domain) pt_dump_seq_printf(st->seq, " %s", st->current_domain); @@ -308,7 +316,7 @@ static void walk_pte(struct pg_state *st, pmd_t *pmd, unsigned long start, for (i = 0; i < PTRS_PER_PTE; i++, pte++) { addr = start + i * PAGE_SIZE; - note_page(st, addr, 4, pte_val(*pte), domain); + note_page(st, addr, 5, pte_val(*pte), domain); } } @@ -341,23 +349,23 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) for (i = 0; i < PTRS_PER_PMD; i++, pmd++) { addr = start + i * PMD_SIZE; domain = get_domain_name(pmd); - if (pmd_none(*pmd) || pmd_large(*pmd) || !pmd_present(*pmd)) - note_page(st, addr, 3, pmd_val(*pmd), domain); + if (pmd_none(*pmd) || pmd_leaf(*pmd) || !pmd_present(*pmd)) + note_page(st, addr, 4, pmd_val(*pmd), domain); else walk_pte(st, pmd, addr, domain); - if (SECTION_SIZE < PMD_SIZE && pmd_large(pmd[1])) { + if (SECTION_SIZE < PMD_SIZE && pmd_leaf(pmd[1])) { addr += SECTION_SIZE; pmd++; domain = get_domain_name(pmd); - note_page(st, addr, 3, pmd_val(*pmd), domain); + note_page(st, addr, 4, pmd_val(*pmd), domain); } } } -static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) +static void walk_pud(struct pg_state *st, p4d_t *p4d, unsigned long start) { - pud_t *pud = pud_offset(pgd, 0); + pud_t *pud = pud_offset(p4d, 0); unsigned long addr; unsigned i; @@ -366,7 +374,23 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) if (!pud_none(*pud)) { walk_pmd(st, pud, addr); } else { - note_page(st, addr, 2, pud_val(*pud), NULL); + note_page(st, addr, 3, pud_val(*pud), NULL); + } + } +} + +static void walk_p4d(struct pg_state *st, pgd_t *pgd, unsigned long start) +{ + p4d_t *p4d = p4d_offset(pgd, 0); + unsigned long addr; + unsigned i; + + for (i = 0; i < PTRS_PER_P4D; i++, p4d++) { + addr = start + i * P4D_SIZE; + if (!p4d_none(*p4d)) { + walk_pud(st, p4d, addr); + } else { + note_page(st, addr, 2, p4d_val(*p4d), NULL); } } } @@ -381,7 +405,7 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm, for (i = 0; i < PTRS_PER_PGD; i++, pgd++) { addr = start + i * PGDIR_SIZE; if (!pgd_none(*pgd)) { - walk_pud(st, pgd, addr); + walk_p4d(st, pgd, addr); } else { note_page(st, addr, 1, pgd_val(*pgd), NULL); } @@ -400,7 +424,7 @@ void ptdump_walk_pgd(struct seq_file *m, struct ptdump_info *info) note_page(&st, 0, 0, 0, NULL); } -static void ptdump_initialize(void) +static void __init ptdump_initialize(void) { unsigned i, j; @@ -413,8 +437,11 @@ static void ptdump_initialize(void) if (pg_level[i].bits[j].nx_bit) pg_level[i].nx_bit = &pg_level[i].bits[j]; } - +#ifdef CONFIG_KASAN + address_markers[4].start_address = VMALLOC_START; +#else address_markers[2].start_address = VMALLOC_START; +#endif } static struct ptdump_info kernel_ptdump_info = { @@ -443,7 +470,7 @@ void ptdump_check_wx(void) pr_info("Checked W+X mappings: passed, no W+X pages found\n"); } -static int ptdump_init(void) +static int __init ptdump_init(void) { ptdump_initialize(); ptdump_debugfs_register(&kernel_ptdump_info, "kernel_page_tables"); diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index ae857f41f68d..39fd5df73317 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -17,7 +17,6 @@ #include <asm/bugs.h> #include <asm/cacheflush.h> #include <asm/cachetype.h> -#include <asm/pgtable.h> #include <asm/tlbflush.h> #include "mm.h" @@ -62,45 +61,27 @@ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address, return ret; } -#if USE_SPLIT_PTE_PTLOCKS -/* - * If we are using split PTE locks, then we need to take the page - * lock here. Otherwise we are using shared mm->page_table_lock - * which is already locked, thus cannot take it. - */ -static inline void do_pte_lock(spinlock_t *ptl) -{ - /* - * Use nested version here to indicate that we are already - * holding one similar spinlock. - */ - spin_lock_nested(ptl, SINGLE_DEPTH_NESTING); -} - -static inline void do_pte_unlock(spinlock_t *ptl) -{ - spin_unlock(ptl); -} -#else /* !USE_SPLIT_PTE_PTLOCKS */ -static inline void do_pte_lock(spinlock_t *ptl) {} -static inline void do_pte_unlock(spinlock_t *ptl) {} -#endif /* USE_SPLIT_PTE_PTLOCKS */ - static int adjust_pte(struct vm_area_struct *vma, unsigned long address, - unsigned long pfn) + unsigned long pfn, bool need_lock) { spinlock_t *ptl; pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte; + pmd_t pmdval; int ret; pgd = pgd_offset(vma->vm_mm, address); if (pgd_none_or_clear_bad(pgd)) return 0; - pud = pud_offset(pgd, address); + p4d = p4d_offset(pgd, address); + if (p4d_none_or_clear_bad(p4d)) + return 0; + + pud = pud_offset(p4d, address); if (pud_none_or_clear_bad(pud)) return 0; @@ -108,18 +89,32 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address, if (pmd_none_or_clear_bad(pmd)) return 0; +again: /* * This is called while another page table is mapped, so we * must use the nested version. This also means we need to * open-code the spin-locking. */ - ptl = pte_lockptr(vma->vm_mm, pmd); - pte = pte_offset_map(pmd, address); - do_pte_lock(ptl); + pte = pte_offset_map_rw_nolock(vma->vm_mm, pmd, address, &pmdval, &ptl); + if (!pte) + return 0; + + if (need_lock) { + /* + * Use nested version here to indicate that we are already + * holding one similar spinlock. + */ + spin_lock_nested(ptl, SINGLE_DEPTH_NESTING); + if (unlikely(!pmd_same(pmdval, pmdp_get_lockless(pmd)))) { + pte_unmap_unlock(pte, ptl); + goto again; + } + } ret = do_adjust_pte(vma, address, pfn, pte); - do_pte_unlock(ptl); + if (need_lock) + spin_unlock(ptl); pte_unmap(pte); return ret; @@ -127,8 +122,10 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address, static void make_coherent(struct address_space *mapping, struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep, unsigned long pfn) + unsigned long addr, pte_t *ptep, unsigned long pfn) { + const unsigned long pmd_start_addr = ALIGN_DOWN(addr, PMD_SIZE); + const unsigned long pmd_end_addr = pmd_start_addr + PMD_SIZE; struct mm_struct *mm = vma->vm_mm; struct vm_area_struct *mpnt; unsigned long offset; @@ -145,6 +142,14 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, flush_dcache_mmap_lock(mapping); vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) { /* + * If we are using split PTE locks, then we need to take the pte + * lock. Otherwise we are using shared mm->page_table_lock which + * is already locked, thus cannot take it. + */ + bool need_lock = IS_ENABLED(CONFIG_SPLIT_PTE_PTLOCKS); + unsigned long mpnt_addr; + + /* * If this VMA is not in our MM, we can ignore it. * Note that we intentionally mask out the VMA * that we are fixing up. @@ -154,7 +159,12 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, if (!(mpnt->vm_flags & VM_MAYSHARE)) continue; offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT; - aliases += adjust_pte(mpnt, mpnt->vm_start + offset, pfn); + mpnt_addr = mpnt->vm_start + offset; + + /* Avoid deadlocks by not grabbing the same PTE lock again. */ + if (mpnt_addr >= pmd_start_addr && mpnt_addr < pmd_end_addr) + need_lock = false; + aliases += adjust_pte(mpnt, mpnt_addr, pfn, need_lock); } flush_dcache_mmap_unlock(mapping); if (aliases) @@ -174,12 +184,12 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, * * Note that the pte lock will be held. */ -void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, - pte_t *ptep) +void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, unsigned int nr) { unsigned long pfn = pte_pfn(*ptep); struct address_space *mapping; - struct page *page; + struct folio *folio; if (!pfn_valid(pfn)) return; @@ -188,13 +198,13 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, * The zero page is never written to, so never has any dirty * cache lines, and therefore never needs to be flushed. */ - page = pfn_to_page(pfn); - if (page == ZERO_PAGE(0)) + if (is_zero_pfn(pfn)) return; - mapping = page_mapping_file(page); - if (!test_and_set_bit(PG_dcache_clean, &page->flags)) - __flush_dcache_page(mapping, page); + folio = page_folio(pfn_to_page(pfn)); + mapping = folio_flush_mapping(folio); + if (!test_and_set_bit(PG_dcache_clean, &folio->flags)) + __flush_dcache_folio(mapping, folio); if (mapping) { if (cache_is_vivt()) make_coherent(mapping, vma, addr, ptep, pfn); diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index bd0f4821f7e1..ab01b51de559 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -17,8 +17,8 @@ #include <linux/sched/debug.h> #include <linux/highmem.h> #include <linux/perf_event.h> +#include <linux/kfence.h> -#include <asm/pgtable.h> #include <asm/system_misc.h> #include <asm/system_info.h> #include <asm/tlbflush.h> @@ -27,6 +27,13 @@ #ifdef CONFIG_MMU +bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size) +{ + unsigned long addr = (unsigned long)unsafe_src; + + return addr >= TASK_SIZE && ULONG_MAX - addr >= size; +} + /* * This is useful to dump out the page tables associated with * 'addr' in mm 'mm'. @@ -38,24 +45,25 @@ void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr) if (!mm) mm = &init_mm; - printk("%spgd = %p\n", lvl, mm->pgd); pgd = pgd_offset(mm, addr); printk("%s[%08lx] *pgd=%08llx", lvl, addr, (long long)pgd_val(*pgd)); do { + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte; - if (pgd_none(*pgd)) + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) break; - if (pgd_bad(*pgd)) { + if (p4d_bad(*p4d)) { pr_cont("(bad)"); break; } - pud = pud_offset(pgd, addr); + pud = pud_offset(p4d, addr); if (PTRS_PER_PUD != 1) pr_cont(", *pud=%08llx", (long long)pud_val(*pud)); @@ -84,6 +92,9 @@ void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr) break; pte = pte_offset_map(pmd, addr); + if (!pte) + break; + pr_cont(", *pte=%08llx", (long long)pte_val(*pte)); #ifndef CONFIG_ARM_LPAE pr_cont(", *ppte=%08llx", @@ -99,6 +110,40 @@ void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr) { } #endif /* CONFIG_MMU */ +static inline bool is_write_fault(unsigned int fsr) +{ + return (fsr & FSR_WRITE) && !(fsr & FSR_CM); +} + +static inline bool is_translation_fault(unsigned int fsr) +{ + int fs = fsr_fs(fsr); +#ifdef CONFIG_ARM_LPAE + if ((fs & FS_MMU_NOLL_MASK) == FS_TRANS_NOLL) + return true; +#else + if (fs == FS_L1_TRANS || fs == FS_L2_TRANS) + return true; +#endif + return false; +} + +static void die_kernel_fault(const char *msg, struct mm_struct *mm, + unsigned long addr, unsigned int fsr, + struct pt_regs *regs) +{ + bust_spinlocks(1); + pr_alert("8<--- cut here ---\n"); + pr_alert("Unable to handle kernel %s at virtual address %08lx when %s\n", + msg, addr, fsr & FSR_LNX_PF ? "execute" : + fsr & FSR_WRITE ? "write" : "read"); + + show_pte(KERN_ALERT, mm, addr); + die("Oops", regs, fsr); + bust_spinlocks(0); + make_task_dead(SIGKILL); +} + /* * Oops. The kernel tried to access some page that wasn't present. */ @@ -106,6 +151,7 @@ static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr, struct pt_regs *regs) { + const char *msg; /* * Are we prepared to handle this kernel fault? */ @@ -115,16 +161,17 @@ __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr, /* * No handler, we'll have to terminate things with extreme prejudice. */ - bust_spinlocks(1); - pr_alert("8<--- cut here ---\n"); - pr_alert("Unable to handle kernel %s at virtual address %08lx\n", - (addr < PAGE_SIZE) ? "NULL pointer dereference" : - "paging request", addr); + if (addr < PAGE_SIZE) { + msg = "NULL pointer dereference"; + } else { + if (is_translation_fault(fsr) && + kfence_handle_page_fault(addr, is_write_fault(fsr), regs)) + return; - show_pte(KERN_ALERT, mm, addr); - die("Oops", regs, fsr); - bust_spinlocks(0); - do_exit(SIGKILL); + msg = "paging request"; + } + + die_kernel_fault(msg, mm, addr, fsr, regs); } /* @@ -179,75 +226,53 @@ void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs) } #ifdef CONFIG_MMU -#define VM_FAULT_BADMAP 0x010000 -#define VM_FAULT_BADACCESS 0x020000 - -/* - * Check that the permissions on the VMA allow for the fault which occurred. - * If we encountered a write fault, we must have write permission, otherwise - * we allow any permission. - */ -static inline bool access_error(unsigned int fsr, struct vm_area_struct *vma) +static inline bool is_permission_fault(unsigned int fsr) { - unsigned int mask = VM_READ | VM_WRITE | VM_EXEC; - - if ((fsr & FSR_WRITE) && !(fsr & FSR_CM)) - mask = VM_WRITE; - if (fsr & FSR_LNX_PF) - mask = VM_EXEC; - - return vma->vm_flags & mask ? false : true; + int fs = fsr_fs(fsr); +#ifdef CONFIG_ARM_LPAE + if ((fs & FS_MMU_NOLL_MASK) == FS_PERM_NOLL) + return true; +#else + if (fs == FS_L1_PERM || fs == FS_L2_PERM) + return true; +#endif + return false; } -static vm_fault_t __kprobes -__do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr, - unsigned int flags, struct task_struct *tsk) +#ifdef CONFIG_CPU_TTBR0_PAN +static inline bool ttbr0_usermode_access_allowed(struct pt_regs *regs) { - struct vm_area_struct *vma; - vm_fault_t fault; + struct svc_pt_regs *svcregs; - vma = find_vma(mm, addr); - fault = VM_FAULT_BADMAP; - if (unlikely(!vma)) - goto out; - if (unlikely(vma->vm_start > addr)) - goto check_stack; - - /* - * Ok, we have a good vm_area for this - * memory access, so we can handle it. - */ -good_area: - if (access_error(fsr, vma)) { - fault = VM_FAULT_BADACCESS; - goto out; - } + /* If we are in user mode: permission granted */ + if (user_mode(regs)) + return true; - return handle_mm_fault(vma, addr & PAGE_MASK, flags); + /* uaccess state saved above pt_regs on SVC exception entry */ + svcregs = to_svc_pt_regs(regs); -check_stack: - /* Don't allow expansion below FIRST_USER_ADDRESS */ - if (vma->vm_flags & VM_GROWSDOWN && - addr >= FIRST_USER_ADDRESS && !expand_stack(vma, addr)) - goto good_area; -out: - return fault; + return !(svcregs->ttbcr & TTBCR_EPD0); } +#else +static inline bool ttbr0_usermode_access_allowed(struct pt_regs *regs) +{ + return true; +} +#endif static int __kprobes do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) { - struct task_struct *tsk; - struct mm_struct *mm; + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; int sig, code; vm_fault_t fault; - unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned int flags = FAULT_FLAG_DEFAULT; + unsigned long vm_flags = VM_ACCESS_FLAGS; if (kprobe_page_fault(regs, fsr)) return 0; - tsk = current; - mm = tsk->mm; /* Enable interrupts if they were enabled in the parent context. */ if (interrupts_enabled(regs)) @@ -262,79 +287,115 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) if (user_mode(regs)) flags |= FAULT_FLAG_USER; - if ((fsr & FSR_WRITE) && !(fsr & FSR_CM)) + + if (is_write_fault(fsr)) { flags |= FAULT_FLAG_WRITE; + vm_flags = VM_WRITE; + } + + if (fsr & FSR_LNX_PF) { + vm_flags = VM_EXEC; + + if (is_permission_fault(fsr) && !user_mode(regs)) + die_kernel_fault("execution of memory", + mm, addr, fsr, regs); + } + + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); /* - * As per x86, we may deadlock here. However, since the kernel only - * validly references user space from well defined areas of the code, - * we can bug out early if this is from code which shouldn't. + * Privileged access aborts with CONFIG_CPU_TTBR0_PAN enabled are + * routed via the translation fault mechanism. Check whether uaccess + * is disabled while in kernel mode. */ - if (!down_read_trylock(&mm->mmap_sem)) { - if (!user_mode(regs) && !search_exception_tables(regs->ARM_pc)) + if (!ttbr0_usermode_access_allowed(regs)) + goto no_context; + + if (!(flags & FAULT_FLAG_USER)) + goto lock_mmap; + + vma = lock_vma_under_rcu(mm, addr); + if (!vma) + goto lock_mmap; + + if (!(vma->vm_flags & vm_flags)) { + vma_end_read(vma); + count_vm_vma_lock_event(VMA_LOCK_SUCCESS); + fault = 0; + code = SEGV_ACCERR; + goto bad_area; + } + fault = handle_mm_fault(vma, addr, flags | FAULT_FLAG_VMA_LOCK, regs); + if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED))) + vma_end_read(vma); + + if (!(fault & VM_FAULT_RETRY)) { + count_vm_vma_lock_event(VMA_LOCK_SUCCESS); + goto done; + } + count_vm_vma_lock_event(VMA_LOCK_RETRY); + if (fault & VM_FAULT_MAJOR) + flags |= FAULT_FLAG_TRIED; + + /* Quick path to respond to signals */ + if (fault_signal_pending(fault, regs)) { + if (!user_mode(regs)) goto no_context; + return 0; + } +lock_mmap: + retry: - down_read(&mm->mmap_sem); - } else { - /* - * The above down_read_trylock() might have succeeded in - * which case, we'll have missed the might_sleep() from - * down_read() - */ - might_sleep(); -#ifdef CONFIG_DEBUG_VM - if (!user_mode(regs) && - !search_exception_tables(regs->ARM_pc)) - goto no_context; -#endif + vma = lock_mm_and_find_vma(mm, addr, regs); + if (unlikely(!vma)) { + fault = 0; + code = SEGV_MAPERR; + goto bad_area; } - fault = __do_page_fault(mm, addr, fsr, flags, tsk); + /* + * ok, we have a good vm_area for this memory access, check the + * permissions on the VMA allow for the fault which occurred. + */ + if (!(vma->vm_flags & vm_flags)) { + mmap_read_unlock(mm); + fault = 0; + code = SEGV_ACCERR; + goto bad_area; + } + + fault = handle_mm_fault(vma, addr & PAGE_MASK, flags, regs); /* If we need to retry but a fatal signal is pending, handle the - * signal first. We do not need to release the mmap_sem because + * signal first. We do not need to release the mmap_lock because * it would already be released in __lock_page_or_retry in * mm/filemap.c. */ - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) { + if (fault_signal_pending(fault, regs)) { if (!user_mode(regs)) goto no_context; return 0; } - /* - * Major/minor page fault accounting is only done on the - * initial attempt. If we go through a retry, it is extremely - * likely that the page will be found in page cache at that point. - */ + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return 0; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); - if (!(fault & VM_FAULT_ERROR) && flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_MAJOR) { - tsk->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, - regs, addr); - } else { - tsk->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, - regs, addr); - } + if (!(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_RETRY) { - /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk - * of starvation. */ - flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; goto retry; } } - up_read(&mm->mmap_sem); + mmap_read_unlock(mm); +done: - /* - * Handle the "normal" case first - VM_FAULT_MAJOR - */ - if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | VM_FAULT_BADACCESS)))) + /* Handle the "normal" case first */ + if (likely(!(fault & VM_FAULT_ERROR))) return 0; + code = SEGV_MAPERR; +bad_area: /* * If we are in kernel mode at this point, we * have no context to handle this fault with. @@ -365,8 +426,6 @@ retry: * isn't in our memory map.. */ sig = SIGSEGV; - code = fault == VM_FAULT_BADACCESS ? - SEGV_ACCERR : SEGV_MAPERR; } __do_user_fault(addr, fsr, sig, code, regs); @@ -408,6 +467,7 @@ do_translation_fault(unsigned long addr, unsigned int fsr, { unsigned int index; pgd_t *pgd, *pgd_k; + p4d_t *p4d, *p4d_k; pud_t *pud, *pud_k; pmd_t *pmd, *pmd_k; @@ -422,13 +482,16 @@ do_translation_fault(unsigned long addr, unsigned int fsr, pgd = cpu_get_pgd() + index; pgd_k = init_mm.pgd + index; - if (pgd_none(*pgd_k)) + p4d = p4d_offset(pgd, addr); + p4d_k = p4d_offset(pgd_k, addr); + + if (p4d_none(*p4d_k)) goto bad_area; - if (!pgd_present(*pgd)) - set_pgd(pgd, *pgd_k); + if (!p4d_present(*p4d)) + set_p4d(p4d, *p4d_k); - pud = pud_offset(pgd, addr); - pud_k = pud_offset(pgd_k, addr); + pud = pud_offset(p4d, addr); + pud_k = pud_offset(p4d_k, addr); if (pud_none(*pud_k)) goto bad_area; @@ -563,6 +626,7 @@ do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs) if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs)) return; + pr_alert("8<--- cut here ---\n"); pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n", inf->name, ifsr, addr); diff --git a/arch/arm/mm/fault.h b/arch/arm/mm/fault.h index 9ecc2097a87a..e8f8c1902544 100644 --- a/arch/arm/mm/fault.h +++ b/arch/arm/mm/fault.h @@ -14,6 +14,9 @@ #ifdef CONFIG_ARM_LPAE #define FSR_FS_AEA 17 +#define FS_TRANS_NOLL 0x4 +#define FS_PERM_NOLL 0xC +#define FS_MMU_NOLL_MASK 0x3C static inline int fsr_fs(unsigned int fsr) { @@ -21,6 +24,10 @@ static inline int fsr_fs(unsigned int fsr) } #else #define FSR_FS_AEA 22 +#define FS_L1_TRANS 0x5 +#define FS_L2_TRANS 0x7 +#define FS_L1_PERM 0xD +#define FS_L2_PERM 0xF static inline int fsr_fs(unsigned int fsr) { @@ -30,5 +37,9 @@ static inline int fsr_fs(unsigned int fsr) void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs); void early_abt_enable(void); +asmlinkage void do_DataAbort(unsigned long addr, unsigned int fsr, + struct pt_regs *regs); +asmlinkage void do_PrefetchAbort(unsigned long addr, unsigned int ifsr, + struct pt_regs *regs); #endif /* __ARCH_ARM_FAULT_H */ diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 6d89db7895d1..0749cf8a6637 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -95,10 +95,10 @@ void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned __flush_icache_all(); } -void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn) +void flush_cache_pages(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn, unsigned int nr) { if (cache_is_vivt()) { - vivt_flush_cache_page(vma, user_addr, pfn); + vivt_flush_cache_pages(vma, user_addr, pfn, nr); return; } @@ -196,29 +196,31 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page, #endif } -void __flush_dcache_page(struct address_space *mapping, struct page *page) +void __flush_dcache_folio(struct address_space *mapping, struct folio *folio) { /* * Writeback any data associated with the kernel mapping of this * page. This ensures that data in the physical page is mutually * coherent with the kernels mapping. */ - if (!PageHighMem(page)) { - __cpuc_flush_dcache_area(page_address(page), page_size(page)); + if (!folio_test_highmem(folio)) { + __cpuc_flush_dcache_area(folio_address(folio), + folio_size(folio)); } else { unsigned long i; if (cache_is_vipt_nonaliasing()) { - for (i = 0; i < compound_nr(page); i++) { - void *addr = kmap_atomic(page + i); + for (i = 0; i < folio_nr_pages(folio); i++) { + void *addr = kmap_local_folio(folio, + i * PAGE_SIZE); __cpuc_flush_dcache_area(addr, PAGE_SIZE); - kunmap_atomic(addr); + kunmap_local(addr); } } else { - for (i = 0; i < compound_nr(page); i++) { - void *addr = kmap_high_get(page + i); + for (i = 0; i < folio_nr_pages(folio); i++) { + void *addr = kmap_high_get(folio_page(folio, i)); if (addr) { __cpuc_flush_dcache_area(addr, PAGE_SIZE); - kunmap_high(page + i); + kunmap_high(folio_page(folio, i)); } } } @@ -230,15 +232,14 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page) * userspace colour, which is congruent with page->index. */ if (mapping && cache_is_vipt_aliasing()) - flush_pfn_alias(page_to_pfn(page), - page->index << PAGE_SHIFT); + flush_pfn_alias(folio_pfn(folio), folio_pos(folio)); } -static void __flush_dcache_aliases(struct address_space *mapping, struct page *page) +static void __flush_dcache_aliases(struct address_space *mapping, struct folio *folio) { struct mm_struct *mm = current->active_mm; - struct vm_area_struct *mpnt; - pgoff_t pgoff; + struct vm_area_struct *vma; + pgoff_t pgoff, pgoff_end; /* * There are possible user space mappings of this page: @@ -246,21 +247,36 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct page *p * data in the current VM view associated with this page. * - aliasing VIPT: we only need to find one mapping of this page. */ - pgoff = page->index; + pgoff = folio->index; + pgoff_end = pgoff + folio_nr_pages(folio) - 1; flush_dcache_mmap_lock(mapping); - vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) { - unsigned long offset; + vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff_end) { + unsigned long start, offset, pfn; + unsigned int nr; /* * If this VMA is not in our MM, we can ignore it. */ - if (mpnt->vm_mm != mm) + if (vma->vm_mm != mm) continue; - if (!(mpnt->vm_flags & VM_MAYSHARE)) + if (!(vma->vm_flags & VM_MAYSHARE)) continue; - offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT; - flush_cache_page(mpnt, mpnt->vm_start + offset, page_to_pfn(page)); + + start = vma->vm_start; + pfn = folio_pfn(folio); + nr = folio_nr_pages(folio); + offset = pgoff - vma->vm_pgoff; + if (offset > -nr) { + pfn -= offset; + nr += offset; + } else { + start += offset * PAGE_SIZE; + } + if (start + nr * PAGE_SIZE > vma->vm_end) + nr = (vma->vm_end - start) / PAGE_SIZE; + + flush_cache_pages(vma, start, pfn, nr); } flush_dcache_mmap_unlock(mapping); } @@ -269,7 +285,7 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct page *p void __sync_icache_dcache(pte_t pteval) { unsigned long pfn; - struct page *page; + struct folio *folio; struct address_space *mapping; if (cache_is_vipt_nonaliasing() && !pte_exec(pteval)) @@ -279,14 +295,17 @@ void __sync_icache_dcache(pte_t pteval) if (!pfn_valid(pfn)) return; - page = pfn_to_page(pfn); + folio = page_folio(pfn_to_page(pfn)); + if (folio_test_reserved(folio)) + return; + if (cache_is_vipt_aliasing()) - mapping = page_mapping_file(page); + mapping = folio_flush_mapping(folio); else mapping = NULL; - if (!test_and_set_bit(PG_dcache_clean, &page->flags)) - __flush_dcache_page(mapping, page); + if (!test_and_set_bit(PG_dcache_clean, &folio->flags)) + __flush_dcache_folio(mapping, folio); if (pte_exec(pteval)) __flush_icache_all(); @@ -312,7 +331,7 @@ void __sync_icache_dcache(pte_t pteval) * Note that we disable the lazy flush for SMP configurations where * the cache maintenance operations are not automatically broadcasted. */ -void flush_dcache_page(struct page *page) +void flush_dcache_folio(struct folio *folio) { struct address_space *mapping; @@ -320,64 +339,36 @@ void flush_dcache_page(struct page *page) * The zero page is never written to, so never has any dirty * cache lines, and therefore never needs to be flushed. */ - if (page == ZERO_PAGE(0)) + if (is_zero_pfn(folio_pfn(folio))) return; if (!cache_ops_need_broadcast() && cache_is_vipt_nonaliasing()) { - if (test_bit(PG_dcache_clean, &page->flags)) - clear_bit(PG_dcache_clean, &page->flags); + if (test_bit(PG_dcache_clean, &folio->flags)) + clear_bit(PG_dcache_clean, &folio->flags); return; } - mapping = page_mapping_file(page); + mapping = folio_flush_mapping(folio); if (!cache_ops_need_broadcast() && - mapping && !page_mapcount(page)) - clear_bit(PG_dcache_clean, &page->flags); + mapping && !folio_mapped(folio)) + clear_bit(PG_dcache_clean, &folio->flags); else { - __flush_dcache_page(mapping, page); + __flush_dcache_folio(mapping, folio); if (mapping && cache_is_vivt()) - __flush_dcache_aliases(mapping, page); + __flush_dcache_aliases(mapping, folio); else if (mapping) __flush_icache_all(); - set_bit(PG_dcache_clean, &page->flags); + set_bit(PG_dcache_clean, &folio->flags); } } -EXPORT_SYMBOL(flush_dcache_page); +EXPORT_SYMBOL(flush_dcache_folio); -/* - * Ensure cache coherency for the kernel mapping of this page. We can - * assume that the page is pinned via kmap. - * - * If the page only exists in the page cache and there are no user - * space mappings, this is a no-op since the page was already marked - * dirty at creation. Otherwise, we need to flush the dirty kernel - * cache lines directly. - */ -void flush_kernel_dcache_page(struct page *page) +void flush_dcache_page(struct page *page) { - if (cache_is_vivt() || cache_is_vipt_aliasing()) { - struct address_space *mapping; - - mapping = page_mapping_file(page); - - if (!mapping || mapping_mapped(mapping)) { - void *addr; - - addr = page_address(page); - /* - * kmap_atomic() doesn't set the page virtual - * address for highmem pages, and - * kunmap_atomic() takes care of cache - * flushing already. - */ - if (!IS_ENABLED(CONFIG_HIGHMEM) || addr) - __cpuc_flush_dcache_area(addr, PAGE_SIZE); - } - } + flush_dcache_folio(page_folio(page)); } -EXPORT_SYMBOL(flush_kernel_dcache_page); - +EXPORT_SYMBOL(flush_dcache_page); /* * Flush an anonymous page so that users of get_user_pages() * can safely access the data. The expected sequence is: @@ -387,6 +378,7 @@ EXPORT_SYMBOL(flush_kernel_dcache_page); * memcpy() to/from page * if written to page, flush_dcache_page() */ +void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr); void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr) { unsigned long pfn; diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c deleted file mode 100644 index a76f8ace9ce6..000000000000 --- a/arch/arm/mm/highmem.c +++ /dev/null @@ -1,146 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * arch/arm/mm/highmem.c -- ARM highmem support - * - * Author: Nicolas Pitre - * Created: september 8, 2008 - * Copyright: Marvell Semiconductors Inc. - */ - -#include <linux/module.h> -#include <linux/highmem.h> -#include <linux/interrupt.h> -#include <asm/fixmap.h> -#include <asm/cacheflush.h> -#include <asm/tlbflush.h> -#include "mm.h" - -static inline void set_fixmap_pte(int idx, pte_t pte) -{ - unsigned long vaddr = __fix_to_virt(idx); - pte_t *ptep = pte_offset_kernel(pmd_off_k(vaddr), vaddr); - - set_pte_ext(ptep, pte, 0); - local_flush_tlb_kernel_page(vaddr); -} - -static inline pte_t get_fixmap_pte(unsigned long vaddr) -{ - pte_t *ptep = pte_offset_kernel(pmd_off_k(vaddr), vaddr); - - return *ptep; -} - -void *kmap(struct page *page) -{ - might_sleep(); - if (!PageHighMem(page)) - return page_address(page); - return kmap_high(page); -} -EXPORT_SYMBOL(kmap); - -void kunmap(struct page *page) -{ - BUG_ON(in_interrupt()); - if (!PageHighMem(page)) - return; - kunmap_high(page); -} -EXPORT_SYMBOL(kunmap); - -void *kmap_atomic(struct page *page) -{ - unsigned int idx; - unsigned long vaddr; - void *kmap; - int type; - - preempt_disable(); - pagefault_disable(); - if (!PageHighMem(page)) - return page_address(page); - -#ifdef CONFIG_DEBUG_HIGHMEM - /* - * There is no cache coherency issue when non VIVT, so force the - * dedicated kmap usage for better debugging purposes in that case. - */ - if (!cache_is_vivt()) - kmap = NULL; - else -#endif - kmap = kmap_high_get(page); - if (kmap) - return kmap; - - type = kmap_atomic_idx_push(); - - idx = FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id(); - vaddr = __fix_to_virt(idx); -#ifdef CONFIG_DEBUG_HIGHMEM - /* - * With debugging enabled, kunmap_atomic forces that entry to 0. - * Make sure it was indeed properly unmapped. - */ - BUG_ON(!pte_none(get_fixmap_pte(vaddr))); -#endif - /* - * When debugging is off, kunmap_atomic leaves the previous mapping - * in place, so the contained TLB flush ensures the TLB is updated - * with the new mapping. - */ - set_fixmap_pte(idx, mk_pte(page, kmap_prot)); - - return (void *)vaddr; -} -EXPORT_SYMBOL(kmap_atomic); - -void __kunmap_atomic(void *kvaddr) -{ - unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; - int idx, type; - - if (kvaddr >= (void *)FIXADDR_START) { - type = kmap_atomic_idx(); - idx = FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id(); - - if (cache_is_vivt()) - __cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE); -#ifdef CONFIG_DEBUG_HIGHMEM - BUG_ON(vaddr != __fix_to_virt(idx)); - set_fixmap_pte(idx, __pte(0)); -#else - (void) idx; /* to kill a warning */ -#endif - kmap_atomic_idx_pop(); - } else if (vaddr >= PKMAP_ADDR(0) && vaddr < PKMAP_ADDR(LAST_PKMAP)) { - /* this address was obtained through kmap_high_get() */ - kunmap_high(pte_page(pkmap_page_table[PKMAP_NR(vaddr)])); - } - pagefault_enable(); - preempt_enable(); -} -EXPORT_SYMBOL(__kunmap_atomic); - -void *kmap_atomic_pfn(unsigned long pfn) -{ - unsigned long vaddr; - int idx, type; - struct page *page = pfn_to_page(pfn); - - preempt_disable(); - pagefault_disable(); - if (!PageHighMem(page)) - return page_address(page); - - type = kmap_atomic_idx_push(); - idx = FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id(); - vaddr = __fix_to_virt(idx); -#ifdef CONFIG_DEBUG_HIGHMEM - BUG_ON(!pte_none(get_fixmap_pte(vaddr))); -#endif - set_fixmap_pte(idx, pfn_pte(pfn, kmap_prot)); - - return (void *)vaddr; -} diff --git a/arch/arm/mm/hugetlbpage.c b/arch/arm/mm/hugetlbpage.c deleted file mode 100644 index a1e5aace897a..000000000000 --- a/arch/arm/mm/hugetlbpage.c +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * arch/arm/mm/hugetlbpage.c - * - * Copyright (C) 2012 ARM Ltd. - * - * Based on arch/x86/include/asm/hugetlb.h and Bill Carson's patches - */ - -#include <linux/init.h> -#include <linux/fs.h> -#include <linux/mm.h> -#include <linux/hugetlb.h> -#include <linux/pagemap.h> -#include <linux/err.h> -#include <linux/sysctl.h> -#include <asm/mman.h> -#include <asm/tlb.h> -#include <asm/tlbflush.h> -#include <asm/pgalloc.h> - -/* - * On ARM, huge pages are backed by pmd's rather than pte's, so we do a lot - * of type casting from pmd_t * to pte_t *. - */ - -int pud_huge(pud_t pud) -{ - return 0; -} - -int pmd_huge(pmd_t pmd) -{ - return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT); -} diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c index a033f6134a64..4a833e89782a 100644 --- a/arch/arm/mm/idmap.c +++ b/arch/arm/mm/idmap.c @@ -3,12 +3,12 @@ #include <linux/kernel.h> #include <linux/slab.h> #include <linux/mm_types.h> +#include <linux/pgtable.h> #include <asm/cputype.h> #include <asm/idmap.h> #include <asm/hwcap.h> #include <asm/pgalloc.h> -#include <asm/pgtable.h> #include <asm/sections.h> #include <asm/system_info.h> @@ -68,7 +68,8 @@ static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end, static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end, unsigned long prot) { - pud_t *pud = pud_offset(pgd, addr); + p4d_t *p4d = p4d_offset(pgd, addr); + pud_t *pud = pud_offset(p4d, addr); unsigned long next; do { @@ -83,8 +84,15 @@ static void identity_mapping_add(pgd_t *pgd, const char *text_start, unsigned long addr, end; unsigned long next; +#ifdef CONFIG_XIP_KERNEL + addr = (phys_addr_t)(text_start) - XIP_VIRT_ADDR(CONFIG_XIP_PHYS_ADDR) + + CONFIG_XIP_PHYS_ADDR; + end = (phys_addr_t)(text_end) - XIP_VIRT_ADDR(CONFIG_XIP_PHYS_ADDR) + + CONFIG_XIP_PHYS_ADDR; +#else addr = virt_to_idmap(text_start); end = virt_to_idmap(text_end); +#endif pr_info("Setting up static identity map for 0x%lx - 0x%lx\n", addr, end); prot |= PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AF; diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 3ef204137e73..54bdca025c9f 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -18,15 +18,16 @@ #include <linux/highmem.h> #include <linux/gfp.h> #include <linux/memblock.h> -#include <linux/dma-contiguous.h> +#include <linux/dma-map-ops.h> #include <linux/sizes.h> #include <linux/stop_machine.h> #include <linux/swiotlb.h> +#include <linux/execmem.h> #include <asm/cp15.h> #include <asm/mach-types.h> #include <asm/memblock.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/prom.h> #include <asm/sections.h> #include <asm/setup.h> @@ -92,18 +93,6 @@ EXPORT_SYMBOL(arm_dma_zone_size); */ phys_addr_t arm_dma_limit; unsigned long arm_dma_pfn_limit; - -static void __init arm_adjust_dma_zone(unsigned long *size, unsigned long *hole, - unsigned long dma_size) -{ - if (size[0] <= dma_size) - return; - - size[ZONE_NORMAL] = size[0] - dma_size; - size[ZONE_DMA] = dma_size; - hole[ZONE_NORMAL] = hole[0]; - hole[ZONE_DMA] = 0; -} #endif void __init setup_dma_zone(const struct machine_desc *mdesc) @@ -121,67 +110,38 @@ void __init setup_dma_zone(const struct machine_desc *mdesc) static void __init zone_sizes_init(unsigned long min, unsigned long max_low, unsigned long max_high) { - unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES]; - struct memblock_region *reg; - - /* - * initialise the zones. - */ - memset(zone_size, 0, sizeof(zone_size)); + unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 }; - /* - * The memory size has already been determined. If we need - * to do anything fancy with the allocation of this memory - * to the zones, now is the time to do it. - */ - zone_size[0] = max_low - min; -#ifdef CONFIG_HIGHMEM - zone_size[ZONE_HIGHMEM] = max_high - max_low; +#ifdef CONFIG_ZONE_DMA + max_zone_pfn[ZONE_DMA] = min(arm_dma_pfn_limit, max_low); #endif - - /* - * Calculate the size of the holes. - * holes = node_size - sum(bank_sizes) - */ - memcpy(zhole_size, zone_size, sizeof(zhole_size)); - for_each_memblock(memory, reg) { - unsigned long start = memblock_region_memory_base_pfn(reg); - unsigned long end = memblock_region_memory_end_pfn(reg); - - if (start < max_low) { - unsigned long low_end = min(end, max_low); - zhole_size[0] -= low_end - start; - } + max_zone_pfn[ZONE_NORMAL] = max_low; #ifdef CONFIG_HIGHMEM - if (end > max_low) { - unsigned long high_start = max(start, max_low); - zhole_size[ZONE_HIGHMEM] -= end - high_start; - } -#endif - } - -#ifdef CONFIG_ZONE_DMA - /* - * Adjust the sizes according to any special requirements for - * this machine type. - */ - if (arm_dma_zone_size) - arm_adjust_dma_zone(zone_size, zhole_size, - arm_dma_zone_size >> PAGE_SHIFT); + max_zone_pfn[ZONE_HIGHMEM] = max_high; #endif - - free_area_init_node(0, zone_size, min, zhole_size); + free_area_init(max_zone_pfn); } #ifdef CONFIG_HAVE_ARCH_PFN_VALID int pfn_valid(unsigned long pfn) { phys_addr_t addr = __pfn_to_phys(pfn); + unsigned long pageblock_size = PAGE_SIZE * pageblock_nr_pages; if (__phys_to_pfn(addr) != pfn) return 0; - return memblock_is_map_memory(addr); + /* + * If address less than pageblock_size bytes away from a present + * memory chunk there still will be a memory map entry for it + * because we round freed memory map to the pageblock boundaries. + */ + if (memblock_overlaps_region(&memblock.memory, + ALIGN_DOWN(addr, pageblock_size), + pageblock_size)) + return 1; + + return 0; } EXPORT_SYMBOL(pfn_valid); #endif @@ -199,53 +159,12 @@ phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align) panic("Failed to steal %pa bytes at %pS\n", &size, (void *)_RET_IP_); - memblock_free(phys, size); + memblock_phys_free(phys, size); memblock_remove(phys, size); return phys; } -static void __init arm_initrd_init(void) -{ -#ifdef CONFIG_BLK_DEV_INITRD - phys_addr_t start; - unsigned long size; - - initrd_start = initrd_end = 0; - - if (!phys_initrd_size) - return; - - /* - * Round the memory region to page boundaries as per free_initrd_mem() - * This allows us to detect whether the pages overlapping the initrd - * are in use, but more importantly, reserves the entire set of pages - * as we don't want these pages allocated for other purposes. - */ - start = round_down(phys_initrd_start, PAGE_SIZE); - size = phys_initrd_size + (phys_initrd_start - start); - size = round_up(size, PAGE_SIZE); - - if (!memblock_is_region_memory(start, size)) { - pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n", - (u64)start, size); - return; - } - - if (memblock_is_region_reserved(start, size)) { - pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n", - (u64)start, size); - return; - } - - memblock_reserve(start, size); - - /* Now convert initrd to virtual addresses */ - initrd_start = __phys_to_virt(phys_initrd_start); - initrd_end = initrd_start + phys_initrd_size; -#endif -} - #ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND void check_cpu_icache_size(int cpuid) { @@ -267,7 +186,7 @@ void __init arm_memblock_init(const struct machine_desc *mdesc) /* Register the kernel text, kernel data and initrd with memblock. */ memblock_reserve(__pa(KERNEL_START), KERNEL_END - KERNEL_START); - arm_initrd_init(); + reserve_initrd_mem(); arm_mm_memblock_reserve(); @@ -275,7 +194,6 @@ void __init arm_memblock_init(const struct machine_desc *mdesc) if (mdesc->reserve) mdesc->reserve(); - early_init_fdt_reserve_self(); early_init_fdt_scan_reserved_mem(); /* reserve memory for DMA contiguous allocations */ @@ -295,18 +213,13 @@ void __init bootmem_init(void) (phys_addr_t)max_low_pfn << PAGE_SHIFT); /* - * Sparsemem tries to allocate bootmem in memory_present(), - * so must be done after the fixed reservations - */ - memblocks_present(); - - /* - * sparse_init() needs the bootmem allocator up and running. + * sparse_init() tries to allocate memory from memblock, so must be + * done after the fixed reservations */ sparse_init(); /* - * Now free the memory - free_area_init_node needs + * Now free the memory - free_area_init needs * the sparse mem_map arrays initialized by sparse_init() * for memmap_init_zone(), otherwise all PFNs are invalid. */ @@ -324,171 +237,17 @@ static inline void poison_init_mem(void *s, size_t count) *p++ = 0xe7fddef0; } -static inline void -free_memmap(unsigned long start_pfn, unsigned long end_pfn) -{ - struct page *start_pg, *end_pg; - phys_addr_t pg, pgend; - - /* - * Convert start_pfn/end_pfn to a struct page pointer. - */ - start_pg = pfn_to_page(start_pfn - 1) + 1; - end_pg = pfn_to_page(end_pfn - 1) + 1; - - /* - * Convert to physical addresses, and - * round start upwards and end downwards. - */ - pg = PAGE_ALIGN(__pa(start_pg)); - pgend = __pa(end_pg) & PAGE_MASK; - - /* - * If there are free pages between these, - * free the section of the memmap array. - */ - if (pg < pgend) - memblock_free_early(pg, pgend - pg); -} - -/* - * The mem_map array can get very big. Free the unused area of the memory map. - */ -static void __init free_unused_memmap(void) -{ - unsigned long start, prev_end = 0; - struct memblock_region *reg; - - /* - * This relies on each bank being in address order. - * The banks are sorted previously in bootmem_init(). - */ - for_each_memblock(memory, reg) { - start = memblock_region_memory_base_pfn(reg); - -#ifdef CONFIG_SPARSEMEM - /* - * Take care not to free memmap entries that don't exist - * due to SPARSEMEM sections which aren't present. - */ - start = min(start, - ALIGN(prev_end, PAGES_PER_SECTION)); -#else - /* - * Align down here since the VM subsystem insists that the - * memmap entries are valid from the bank start aligned to - * MAX_ORDER_NR_PAGES. - */ - start = round_down(start, MAX_ORDER_NR_PAGES); -#endif - /* - * If we had a previous bank, and there is a space - * between the current bank and the previous, free it. - */ - if (prev_end && prev_end < start) - free_memmap(prev_end, start); - - /* - * Align up here since the VM subsystem insists that the - * memmap entries are valid from the bank end aligned to - * MAX_ORDER_NR_PAGES. - */ - prev_end = ALIGN(memblock_region_memory_end_pfn(reg), - MAX_ORDER_NR_PAGES); - } - -#ifdef CONFIG_SPARSEMEM - if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) - free_memmap(prev_end, - ALIGN(prev_end, PAGES_PER_SECTION)); -#endif -} - -#ifdef CONFIG_HIGHMEM -static inline void free_area_high(unsigned long pfn, unsigned long end) -{ - for (; pfn < end; pfn++) - free_highmem_page(pfn_to_page(pfn)); -} -#endif - -static void __init free_highpages(void) -{ -#ifdef CONFIG_HIGHMEM - unsigned long max_low = max_low_pfn; - struct memblock_region *mem, *res; - - /* set highmem page free */ - for_each_memblock(memory, mem) { - unsigned long start = memblock_region_memory_base_pfn(mem); - unsigned long end = memblock_region_memory_end_pfn(mem); - - /* Ignore complete lowmem entries */ - if (end <= max_low) - continue; - - if (memblock_is_nomap(mem)) - continue; - - /* Truncate partial highmem entries */ - if (start < max_low) - start = max_low; - - /* Find and exclude any reserved regions */ - for_each_memblock(reserved, res) { - unsigned long res_start, res_end; - - res_start = memblock_region_reserved_base_pfn(res); - res_end = memblock_region_reserved_end_pfn(res); - - if (res_end < start) - continue; - if (res_start < start) - res_start = start; - if (res_start > end) - res_start = end; - if (res_end > end) - res_end = end; - if (res_start != start) - free_area_high(start, res_start); - start = res_end; - if (start == end) - break; - } - - /* And now free anything which remains */ - if (start < end) - free_area_high(start, end); - } -#endif -} - -/* - * mem_init() marks the free areas in the mem_map and tells us how much - * memory is free. This is done after various parts of the system have - * claimed their memory after the kernel image. - */ -void __init mem_init(void) +void __init arch_mm_preinit(void) { #ifdef CONFIG_ARM_LPAE - swiotlb_init(1); + swiotlb_init(max_pfn > arm_dma_pfn_limit, SWIOTLB_VERBOSE); #endif - set_max_mapnr(pfn_to_page(max_pfn) - mem_map); - - /* this will put all unused low memory onto the freelists */ - free_unused_memmap(); - memblock_free_all(); - #ifdef CONFIG_SA1111 /* now that our DMA memory is actually so designated, we can free it */ - free_reserved_area(__va(PHYS_OFFSET), swapper_pg_dir, -1, NULL); + memblock_phys_free(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET); #endif - free_highpages(); - - mem_init_print_info(NULL); - /* * Check boundaries twice: Some fundamental inconsistencies can * be detected at build time already. @@ -571,7 +330,7 @@ static inline void section_update(unsigned long addr, pmdval_t mask, { pmd_t *pmd; - pmd = pmd_offset(pud_offset(pgd_offset(mm, addr), addr), addr); + pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, addr), addr), addr), addr); #ifdef CONFIG_ARM_LPAE pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot); @@ -621,7 +380,7 @@ static void set_section_perms(struct section_perm *perms, int n, bool set, } -/** +/* * update_sections_early intended to be called only through stop_machine * framework and executed by only one CPU while all other CPUs will spin and * wait, so no locking is required in this function. @@ -658,31 +417,10 @@ static int __mark_rodata_ro(void *unused) return 0; } -static int kernel_set_to_readonly __read_mostly; - void mark_rodata_ro(void) { - kernel_set_to_readonly = 1; stop_machine(__mark_rodata_ro, NULL, NULL); - debug_checkwx(); -} - -void set_kernel_text_rw(void) -{ - if (!kernel_set_to_readonly) - return; - - set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), false, - current->active_mm); -} - -void set_kernel_text_ro(void) -{ - if (!kernel_set_to_readonly) - return; - - set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), true, - current->active_mm); + arm_debug_checkwx(); } #else @@ -710,3 +448,47 @@ void free_initrd_mem(unsigned long start, unsigned long end) free_reserved_area((void *)start, (void *)end, -1, "initrd"); } #endif + +#ifdef CONFIG_EXECMEM + +#ifdef CONFIG_XIP_KERNEL +/* + * The XIP kernel text is mapped in the module area for modules and + * some other stuff to work without any indirect relocations. + * MODULES_VADDR is redefined here and not in asm/memory.h to avoid + * recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off. + */ +#undef MODULES_VADDR +#define MODULES_VADDR (((unsigned long)_exiprom + ~PMD_MASK) & PMD_MASK) +#endif + +#ifdef CONFIG_MMU +static struct execmem_info execmem_info __ro_after_init; + +struct execmem_info __init *execmem_arch_setup(void) +{ + unsigned long fallback_start = 0, fallback_end = 0; + + if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS)) { + fallback_start = VMALLOC_START; + fallback_end = VMALLOC_END; + } + + execmem_info = (struct execmem_info){ + .ranges = { + [EXECMEM_DEFAULT] = { + .start = MODULES_VADDR, + .end = MODULES_END, + .pgprot = PAGE_KERNEL_EXEC, + .alignment = 1, + .fallback_start = fallback_start, + .fallback_end = fallback_end, + }, + }, + }; + + return &execmem_info; +} +#endif /* CONFIG_MMU */ + +#endif /* CONFIG_EXECMEM */ diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index 72286f9a4d30..27e64f782cb3 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -23,10 +23,12 @@ */ #include <linux/module.h> #include <linux/errno.h> +#include <linux/kasan.h> #include <linux/mm.h> #include <linux/vmalloc.h> #include <linux/io.h> #include <linux/sizes.h> +#include <linux/memblock.h> #include <asm/cp15.h> #include <asm/cputype.h> @@ -35,6 +37,7 @@ #include <asm/mmu_context.h> #include <asm/pgalloc.h> #include <asm/tlbflush.h> +#include <asm/set_memory.h> #include <asm/system_info.h> #include <asm/mach/map.h> @@ -108,23 +111,52 @@ void __init add_static_vm_early(struct static_vm *svm) int ioremap_page(unsigned long virt, unsigned long phys, const struct mem_type *mtype) { - return ioremap_page_range(virt, virt + PAGE_SIZE, phys, - __pgprot(mtype->prot_pte)); + return vmap_page_range(virt, virt + PAGE_SIZE, phys, + __pgprot(mtype->prot_pte)); } EXPORT_SYMBOL(ioremap_page); +#ifdef CONFIG_KASAN +static unsigned long arm_kasan_mem_to_shadow(unsigned long addr) +{ + return (unsigned long)kasan_mem_to_shadow((void *)addr); +} +#else +static unsigned long arm_kasan_mem_to_shadow(unsigned long addr) +{ + return 0; +} +#endif + +static void memcpy_pgd(struct mm_struct *mm, unsigned long start, + unsigned long end) +{ + end = ALIGN(end, PGDIR_SIZE); + memcpy(pgd_offset(mm, start), pgd_offset_k(start), + sizeof(pgd_t) * (pgd_index(end) - pgd_index(start))); +} + void __check_vmalloc_seq(struct mm_struct *mm) { - unsigned int seq; + int seq; do { - seq = init_mm.context.vmalloc_seq; - memcpy(pgd_offset(mm, VMALLOC_START), - pgd_offset_k(VMALLOC_START), - sizeof(pgd_t) * (pgd_index(VMALLOC_END) - - pgd_index(VMALLOC_START))); - mm->context.vmalloc_seq = seq; - } while (seq != init_mm.context.vmalloc_seq); + seq = atomic_read_acquire(&init_mm.context.vmalloc_seq); + memcpy_pgd(mm, VMALLOC_START, VMALLOC_END); + if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) { + unsigned long start = + arm_kasan_mem_to_shadow(VMALLOC_START); + unsigned long end = + arm_kasan_mem_to_shadow(VMALLOC_END); + memcpy_pgd(mm, start, end); + } + /* + * Use a store-release so that other CPUs that observe the + * counter's new value are guaranteed to see the results of the + * memcpy as well. + */ + atomic_set_release(&mm->context.vmalloc_seq, seq); + } while (seq != atomic_read(&init_mm.context.vmalloc_seq)); } #if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE) @@ -141,14 +173,8 @@ void __check_vmalloc_seq(struct mm_struct *mm) static void unmap_area_sections(unsigned long virt, unsigned long size) { unsigned long addr = virt, end = virt + (size & ~(SZ_1M - 1)); - pgd_t *pgd; - pud_t *pud; - pmd_t *pmdp; - - flush_cache_vunmap(addr, end); - pgd = pgd_offset_k(addr); - pud = pud_offset(pgd, addr); - pmdp = pmd_offset(pud, addr); + pmd_t *pmdp = pmd_off_k(addr); + do { pmd_t pmd = *pmdp; @@ -161,7 +187,7 @@ static void unmap_area_sections(unsigned long virt, unsigned long size) * Note: this is still racy on SMP machines. */ pmd_clear(pmdp); - init_mm.context.vmalloc_seq++; + atomic_inc_return_release(&init_mm.context.vmalloc_seq); /* * Free the page table, if there was one. @@ -178,8 +204,7 @@ static void unmap_area_sections(unsigned long virt, unsigned long size) * Ensure that the active_mm is up to date - we want to * catch any use-after-iounmap cases. */ - if (current->active_mm->context.vmalloc_seq != init_mm.context.vmalloc_seq) - __check_vmalloc_seq(current->active_mm); + check_vmalloc_seq(current->active_mm); flush_tlb_kernel_range(virt, end); } @@ -189,9 +214,7 @@ remap_area_sections(unsigned long virt, unsigned long pfn, size_t size, const struct mem_type *type) { unsigned long addr = virt, end = virt + size; - pgd_t *pgd; - pud_t *pud; - pmd_t *pmd; + pmd_t *pmd = pmd_off_k(addr); /* * Remove and free any PTE-based mapping, and @@ -199,9 +222,6 @@ remap_area_sections(unsigned long virt, unsigned long pfn, */ unmap_area_sections(virt, size); - pgd = pgd_offset_k(addr); - pud = pud_offset(pgd, addr); - pmd = pmd_offset(pud, addr); do { pmd[0] = __pmd(__pfn_to_phys(pfn) | type->prot_sect); pfn += SZ_1M >> PAGE_SHIFT; @@ -221,19 +241,13 @@ remap_area_supersections(unsigned long virt, unsigned long pfn, size_t size, const struct mem_type *type) { unsigned long addr = virt, end = virt + size; - pgd_t *pgd; - pud_t *pud; - pmd_t *pmd; + pmd_t *pmd = pmd_off_k(addr); /* * Remove and free any PTE-based mapping, and * sync the current kernel mapping. */ unmap_area_sections(virt, size); - - pgd = pgd_offset_k(virt); - pud = pud_offset(pgd, addr); - pmd = pmd_offset(pud, addr); do { unsigned long super_pmd_val, i; @@ -301,7 +315,8 @@ static void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn, * Don't allow RAM to be mapped with mismatched attributes - this * causes problems with ARMv6+ */ - if (WARN_ON(pfn_valid(pfn) && mtype != MT_MEMORY_RW)) + if (WARN_ON(memblock_is_map_memory(PFN_PHYS(pfn)) && + mtype != MT_MEMORY_RW)) return NULL; area = get_vm_area_caller(size, VM_IOREMAP, caller); @@ -416,14 +431,19 @@ __arm_ioremap_exec(phys_addr_t phys_addr, size_t size, bool cached) __builtin_return_address(0)); } -void *arch_memremap_wb(phys_addr_t phys_addr, size_t size) +void __arm_iomem_set_ro(void __iomem *ptr, size_t size) +{ + set_memory_ro((unsigned long)ptr, PAGE_ALIGN(size) / PAGE_SIZE); +} + +void *arch_memremap_wb(phys_addr_t phys_addr, size_t size, unsigned long flags) { return (__force void *)arch_ioremap_caller(phys_addr, size, MT_MEMORY_RW, __builtin_return_address(0)); } -void __iounmap(volatile void __iomem *io_addr) +void iounmap(volatile void __iomem *io_addr) { void *addr = (void *)(PAGE_MASK & (unsigned long)io_addr); struct static_vm *svm; @@ -451,16 +471,9 @@ void __iounmap(volatile void __iomem *io_addr) vunmap(addr); } - -void (*arch_iounmap)(volatile void __iomem *) = __iounmap; - -void iounmap(volatile void __iomem *cookie) -{ - arch_iounmap(cookie); -} EXPORT_SYMBOL(iounmap); -#ifdef CONFIG_PCI +#if defined(CONFIG_PCI) || IS_ENABLED(CONFIG_PCMCIA) static int pci_ioremap_mem_type = MT_DEVICE; void pci_ioremap_set_mem_type(int mem_type) @@ -468,16 +481,20 @@ void pci_ioremap_set_mem_type(int mem_type) pci_ioremap_mem_type = mem_type; } -int pci_ioremap_io(unsigned int offset, phys_addr_t phys_addr) +int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr) { - BUG_ON(offset + SZ_64K - 1 > IO_SPACE_LIMIT); + unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start; + + if (!(res->flags & IORESOURCE_IO)) + return -EINVAL; + + if (res->end > IO_SPACE_LIMIT) + return -EINVAL; - return ioremap_page_range(PCI_IO_VIRT_BASE + offset, - PCI_IO_VIRT_BASE + offset + SZ_64K, - phys_addr, - __pgprot(get_mem_type(pci_ioremap_mem_type)->prot_pte)); + return vmap_page_range(vaddr, vaddr + resource_size(res), phys_addr, + __pgprot(get_mem_type(pci_ioremap_mem_type)->prot_pte)); } -EXPORT_SYMBOL_GPL(pci_ioremap_io); +EXPORT_SYMBOL(pci_remap_iospace); void __iomem *pci_remap_cfgspace(resource_size_t res_cookie, size_t size) { @@ -494,3 +511,9 @@ void __init early_ioremap_init(void) { early_ioremap_setup(); } + +bool arch_memremap_can_ram_remap(resource_size_t offset, size_t size, + unsigned long flags) +{ + return memblock_is_map_memory(offset); +} diff --git a/arch/arm/mm/kasan_init.c b/arch/arm/mm/kasan_init.c new file mode 100644 index 000000000000..111d4f703136 --- /dev/null +++ b/arch/arm/mm/kasan_init.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file contains kasan initialization code for ARM. + * + * Copyright (c) 2018 Samsung Electronics Co., Ltd. + * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com> + * Author: Linus Walleij <linus.walleij@linaro.org> + */ + +#define pr_fmt(fmt) "kasan: " fmt +#include <linux/kasan.h> +#include <linux/kernel.h> +#include <linux/memblock.h> +#include <linux/sched/task.h> +#include <linux/start_kernel.h> +#include <linux/pgtable.h> +#include <asm/cputype.h> +#include <asm/highmem.h> +#include <asm/mach/map.h> +#include <asm/page.h> +#include <asm/pgalloc.h> +#include <asm/procinfo.h> +#include <asm/proc-fns.h> + +#include "mm.h" + +static pgd_t tmp_pgd_table[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE); + +pmd_t tmp_pmd_table[PTRS_PER_PMD] __page_aligned_bss; + +static __init void *kasan_alloc_block_raw(size_t size) +{ + return memblock_alloc_try_nid_raw(size, size, __pa(MAX_DMA_ADDRESS), + MEMBLOCK_ALLOC_NOLEAKTRACE, NUMA_NO_NODE); +} + +static __init void *kasan_alloc_block(size_t size) +{ + return memblock_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS), + MEMBLOCK_ALLOC_NOLEAKTRACE, NUMA_NO_NODE); +} + +static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr, + unsigned long end, bool early) +{ + unsigned long next; + pte_t *ptep = pte_offset_kernel(pmdp, addr); + + do { + pte_t entry; + void *p; + + next = addr + PAGE_SIZE; + + if (!early) { + if (!pte_none(READ_ONCE(*ptep))) + continue; + + p = kasan_alloc_block_raw(PAGE_SIZE); + if (!p) { + panic("%s failed to allocate shadow page for address 0x%lx\n", + __func__, addr); + return; + } + memset(p, KASAN_SHADOW_INIT, PAGE_SIZE); + entry = pfn_pte(virt_to_pfn(p), + __pgprot(pgprot_val(PAGE_KERNEL))); + } else if (pte_none(READ_ONCE(*ptep))) { + /* + * The early shadow memory is mapping all KASan + * operations to one and the same page in memory, + * "kasan_early_shadow_page" so that the instrumentation + * will work on a scratch area until we can set up the + * proper KASan shadow memory. + */ + entry = pfn_pte(virt_to_pfn(kasan_early_shadow_page), + __pgprot(_L_PTE_DEFAULT | L_PTE_DIRTY | L_PTE_XN)); + } else { + /* + * Early shadow mappings are PMD_SIZE aligned, so if the + * first entry is already set, they must all be set. + */ + return; + } + + set_pte_at(&init_mm, addr, ptep, entry); + } while (ptep++, addr = next, addr != end); +} + +/* + * The pmd (page middle directory) is only used on LPAE + */ +static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, + unsigned long end, bool early) +{ + unsigned long next; + pmd_t *pmdp = pmd_offset(pudp, addr); + + do { + if (pmd_none(*pmdp)) { + /* + * We attempt to allocate a shadow block for the PMDs + * used by the PTEs for this address if it isn't already + * allocated. + */ + void *p = early ? kasan_early_shadow_pte : + kasan_alloc_block(PAGE_SIZE); + + if (!p) { + panic("%s failed to allocate shadow block for address 0x%lx\n", + __func__, addr); + return; + } + pmd_populate_kernel(&init_mm, pmdp, p); + flush_pmd_entry(pmdp); + } + + next = pmd_addr_end(addr, end); + kasan_pte_populate(pmdp, addr, next, early); + } while (pmdp++, addr = next, addr != end); +} + +static void __init kasan_pgd_populate(unsigned long addr, unsigned long end, + bool early) +{ + unsigned long next; + pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; + + pgdp = pgd_offset_k(addr); + + do { + /* + * Allocate and populate the shadow block of p4d folded into + * pud folded into pmd if it doesn't already exist + */ + if (!early && pgd_none(*pgdp)) { + void *p = kasan_alloc_block(PAGE_SIZE); + + if (!p) { + panic("%s failed to allocate shadow block for address 0x%lx\n", + __func__, addr); + return; + } + pgd_populate(&init_mm, pgdp, p); + } + + next = pgd_addr_end(addr, end); + /* + * We just immediately jump over the p4d and pud page + * directories since we believe ARM32 will never gain four + * nor five level page tables. + */ + p4dp = p4d_offset(pgdp, addr); + pudp = pud_offset(p4dp, addr); + + kasan_pmd_populate(pudp, addr, next, early); + } while (pgdp++, addr = next, addr != end); +} + +extern struct proc_info_list *lookup_processor_type(unsigned int); + +void __init kasan_early_init(void) +{ + struct proc_info_list *list; + + /* + * locate processor in the list of supported processor + * types. The linker builds this table for us from the + * entries in arch/arm/mm/proc-*.S + */ + list = lookup_processor_type(read_cpuid_id()); + if (list) { +#ifdef MULTI_CPU + processor = *list->proc; +#endif + } + + BUILD_BUG_ON((KASAN_SHADOW_END - (1UL << 29)) != KASAN_SHADOW_OFFSET); + /* + * We walk the page table and set all of the shadow memory to point + * to the scratch page. + */ + kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, true); +} + +static void __init clear_pgds(unsigned long start, + unsigned long end) +{ + for (; start && start < end; start += PMD_SIZE) + pmd_clear(pmd_off_k(start)); +} + +static int __init create_mapping(void *start, void *end) +{ + void *shadow_start, *shadow_end; + + shadow_start = kasan_mem_to_shadow(start); + shadow_end = kasan_mem_to_shadow(end); + + pr_info("Mapping kernel virtual memory block: %px-%px at shadow: %px-%px\n", + start, end, shadow_start, shadow_end); + + kasan_pgd_populate((unsigned long)shadow_start & PAGE_MASK, + PAGE_ALIGN((unsigned long)shadow_end), false); + return 0; +} + +void __init kasan_init(void) +{ + phys_addr_t pa_start, pa_end; + u64 i; + + /* + * We are going to perform proper setup of shadow memory. + * + * At first we should unmap early shadow (clear_pgds() call bellow). + * However, instrumented code can't execute without shadow memory. + * + * To keep the early shadow memory MMU tables around while setting up + * the proper shadow memory, we copy swapper_pg_dir (the initial page + * table) to tmp_pgd_table and use that to keep the early shadow memory + * mapped until the full shadow setup is finished. Then we swap back + * to the proper swapper_pg_dir. + */ + + memcpy(tmp_pgd_table, swapper_pg_dir, sizeof(tmp_pgd_table)); +#ifdef CONFIG_ARM_LPAE + /* We need to be in the same PGD or this won't work */ + BUILD_BUG_ON(pgd_index(KASAN_SHADOW_START) != + pgd_index(KASAN_SHADOW_END)); + memcpy(tmp_pmd_table, + (void*)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_START)), + sizeof(tmp_pmd_table)); + set_pgd(&tmp_pgd_table[pgd_index(KASAN_SHADOW_START)], + __pgd(__pa(tmp_pmd_table) | PMD_TYPE_TABLE | L_PGD_SWAPPER)); +#endif + cpu_switch_mm(tmp_pgd_table, &init_mm); + local_flush_tlb_all(); + + clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END); + + if (!IS_ENABLED(CONFIG_KASAN_VMALLOC)) + kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_START), + kasan_mem_to_shadow((void *)VMALLOC_END)); + + kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_END), + kasan_mem_to_shadow((void *)-1UL) + 1); + + for_each_mem_range(i, &pa_start, &pa_end) { + void *start = __va(pa_start); + void *end = __va(pa_end); + + /* Do not attempt to shadow highmem */ + if (pa_start >= arm_lowmem_limit) { + pr_info("Skip highmem block at %pa-%pa\n", &pa_start, &pa_end); + continue; + } + if (pa_end > arm_lowmem_limit) { + pr_info("Truncating shadow for memory block at %pa-%pa to lowmem region at %pa\n", + &pa_start, &pa_end, &arm_lowmem_limit); + end = __va(arm_lowmem_limit); + } + if (start >= end) { + pr_info("Skipping invalid memory block %pa-%pa (virtual %p-%p)\n", + &pa_start, &pa_end, start, end); + continue; + } + + create_mapping(start, end); + } + + /* + * 1. The module global variables are in MODULES_VADDR ~ MODULES_END, + * so we need to map this area if CONFIG_KASAN_VMALLOC=n. With + * VMALLOC support KASAN will manage this region dynamically, + * refer to kasan_populate_vmalloc() and ARM's implementation of + * module_alloc(). + * 2. PKMAP_BASE ~ PKMAP_BASE+PMD_SIZE's shadow and MODULES_VADDR + * ~ MODULES_END's shadow is in the same PMD_SIZE, so we can't + * use kasan_populate_zero_shadow. + */ + if (!IS_ENABLED(CONFIG_KASAN_VMALLOC) && IS_ENABLED(CONFIG_MODULES)) + create_mapping((void *)MODULES_VADDR, (void *)(MODULES_END)); + create_mapping((void *)PKMAP_BASE, (void *)(PKMAP_BASE + PMD_SIZE)); + + /* + * KAsan may reuse the contents of kasan_early_shadow_pte directly, so + * we should make sure that it maps the zero page read-only. + */ + for (i = 0; i < PTRS_PER_PTE; i++) + set_pte_at(&init_mm, KASAN_SHADOW_START + i*PAGE_SIZE, + &kasan_early_shadow_pte[i], + pfn_pte(virt_to_pfn(kasan_early_shadow_page), + __pgprot(pgprot_val(PAGE_KERNEL) + | L_PTE_RDONLY))); + + cpu_switch_mm(swapper_pg_dir, &init_mm); + local_flush_tlb_all(); + + memset(kasan_early_shadow_page, 0, PAGE_SIZE); + pr_info("Kernel address sanitizer initialized\n"); + init_task.kasan_depth = 0; +} diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h index 88c121ac14b3..419316316711 100644 --- a/arch/arm/mm/mm.h +++ b/arch/arm/mm/mm.h @@ -2,8 +2,7 @@ #ifdef CONFIG_MMU #include <linux/list.h> #include <linux/vmalloc.h> - -#include <asm/pgtable.h> +#include <linux/pgtable.h> /* the upper-most page table pointer */ extern pmd_t *top_pmd; @@ -36,11 +35,6 @@ static inline pte_t get_top_pte(unsigned long va) return *ptep; } -static inline pmd_t *pmd_off_k(unsigned long virt) -{ - return pmd_offset(pud_offset(pgd_offset_k(virt), virt), virt); -} - struct mem_type { pteval_t prot_pte; pteval_t prot_pte_s2; @@ -51,7 +45,7 @@ struct mem_type { const struct mem_type *get_mem_type(unsigned int type); -extern void __flush_dcache_page(struct address_space *mapping, struct page *page); +void __flush_dcache_folio(struct address_space *mapping, struct folio *folio); /* * ARM specific vm_struct->flags bits. @@ -94,6 +88,10 @@ extern phys_addr_t arm_lowmem_limit; void __init bootmem_init(void); void arm_mm_memblock_reserve(void); +#ifdef CONFIG_CMA_AREAS void dma_contiguous_remap(void); +#else +static inline void dma_contiguous_remap(void) { } +#endif unsigned long __clear_cr(unsigned long mask); diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c index b8d912ac9e61..3dbb383c26d5 100644 --- a/arch/arm/mm/mmap.c +++ b/arch/arm/mm/mmap.c @@ -28,13 +28,14 @@ */ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, - unsigned long len, unsigned long pgoff, unsigned long flags) + unsigned long len, unsigned long pgoff, + unsigned long flags, vm_flags_t vm_flags) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; int do_align = 0; int aliasing = cache_is_vipt_aliasing(); - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; /* * We only need to do colour alignment if either the I or D @@ -68,7 +69,6 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, return addr; } - info.flags = 0; info.length = len; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; @@ -79,15 +79,15 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, - const unsigned long len, const unsigned long pgoff, - const unsigned long flags) + const unsigned long len, const unsigned long pgoff, + const unsigned long flags, vm_flags_t vm_flags) { struct vm_area_struct *vma; struct mm_struct *mm = current->mm; unsigned long addr = addr0; int do_align = 0; int aliasing = cache_is_vipt_aliasing(); - struct vm_unmapped_area_info info; + struct vm_unmapped_area_info info = {}; /* * We only need to do colour alignment if either the I or D @@ -165,25 +165,3 @@ int valid_mmap_phys_addr_range(unsigned long pfn, size_t size) { return (pfn + (size >> PAGE_SHIFT)) <= (1 + (PHYS_MASK >> PAGE_SHIFT)); } - -#ifdef CONFIG_STRICT_DEVMEM - -#include <linux/ioport.h> - -/* - * devmem_is_allowed() checks to see if /dev/mem access to a certain - * address is valid. The argument is a physical page number. - * We mimic x86 here by disallowing access to system RAM as well as - * device-exclusive MMIO regions. This effectively disable read()/write() - * on /dev/mem. - */ -int devmem_is_allowed(unsigned long pfn) -{ - if (iomem_is_exclusive(pfn << PAGE_SHIFT)) - return 0; - if (!page_is_ram(pfn)) - return 1; - return 0; -} - -#endif diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 5d0d0f86e790..f02f872ea8a9 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -17,18 +17,19 @@ #include <asm/cp15.h> #include <asm/cputype.h> -#include <asm/sections.h> #include <asm/cachetype.h> -#include <asm/fixmap.h> #include <asm/sections.h> #include <asm/setup.h> #include <asm/smp_plat.h> +#include <asm/tcm.h> #include <asm/tlb.h> #include <asm/highmem.h> #include <asm/system_info.h> #include <asm/traps.h> #include <asm/procinfo.h> -#include <asm/memory.h> +#include <asm/page.h> +#include <asm/pgalloc.h> +#include <asm/kasan_def.h> #include <asm/mach/arch.h> #include <asm/mach/map.h> @@ -37,7 +38,8 @@ #include "fault.h" #include "mm.h" -#include "tcm.h" + +extern unsigned long __atags_pointer; /* * empty_zero_page is a special page that is used for @@ -63,9 +65,6 @@ static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK; static unsigned int ecc_mask __initdata = 0; pgprot_t pgprot_user; pgprot_t pgprot_kernel; -pgprot_t pgprot_hyp_device; -pgprot_t pgprot_s2; -pgprot_t pgprot_s2_device; EXPORT_SYMBOL(pgprot_user); EXPORT_SYMBOL(pgprot_kernel); @@ -75,48 +74,34 @@ struct cachepolicy { unsigned int cr_mask; pmdval_t pmd; pteval_t pte; - pteval_t pte_s2; }; -#ifdef CONFIG_ARM_LPAE -#define s2_policy(policy) policy -#else -#define s2_policy(policy) 0 -#endif - -unsigned long kimage_voffset __ro_after_init; - static struct cachepolicy cache_policies[] __initdata = { { .policy = "uncached", .cr_mask = CR_W|CR_C, .pmd = PMD_SECT_UNCACHED, .pte = L_PTE_MT_UNCACHED, - .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED), }, { .policy = "buffered", .cr_mask = CR_C, .pmd = PMD_SECT_BUFFERED, .pte = L_PTE_MT_BUFFERABLE, - .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED), }, { .policy = "writethrough", .cr_mask = 0, .pmd = PMD_SECT_WT, .pte = L_PTE_MT_WRITETHROUGH, - .pte_s2 = s2_policy(L_PTE_S2_MT_WRITETHROUGH), }, { .policy = "writeback", .cr_mask = 0, .pmd = PMD_SECT_WB, .pte = L_PTE_MT_WRITEBACK, - .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK), }, { .policy = "writealloc", .cr_mask = 0, .pmd = PMD_SECT_WBWA, .pte = L_PTE_MT_WRITEALLOC, - .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK), } }; @@ -227,12 +212,14 @@ early_param("ecc", early_ecc); static int __init early_cachepolicy(char *p) { pr_warn("cachepolicy kernel parameter not supported without cp15\n"); + return 0; } early_param("cachepolicy", early_cachepolicy); static int __init noalign_setup(char *__unused) { pr_warn("noalign kernel parameter not supported without cp15\n"); + return 1; } __setup("noalign", noalign_setup); @@ -246,9 +233,6 @@ static struct mem_type mem_types[] __ro_after_init = { [MT_DEVICE] = { /* Strongly ordered / ARMv6 shared device */ .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED | L_PTE_SHARED, - .prot_pte_s2 = s2_policy(PROT_PTE_S2_DEVICE) | - s2_policy(L_PTE_S2_MT_DEV_SHARED) | - L_PTE_SHARED, .prot_l1 = PMD_TYPE_TABLE, .prot_sect = PROT_SECT_DEVICE | PMD_SECT_S, .domain = DOMAIN_IO, @@ -312,6 +296,17 @@ static struct mem_type mem_types[] __ro_after_init = { .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE, .domain = DOMAIN_KERNEL, }, + [MT_MEMORY_RO] = { + .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | + L_PTE_XN | L_PTE_RDONLY, + .prot_l1 = PMD_TYPE_TABLE, +#ifdef CONFIG_ARM_LPAE + .prot_sect = PMD_TYPE_SECT | L_PMD_SECT_RDONLY | PMD_SECT_AP2, +#else + .prot_sect = PMD_TYPE_SECT, +#endif + .domain = DOMAIN_KERNEL, + }, [MT_ROM] = { .prot_sect = PMD_TYPE_SECT, .domain = DOMAIN_KERNEL, @@ -374,11 +369,7 @@ static pte_t *pte_offset_late_fixmap(pmd_t *dir, unsigned long addr) static inline pmd_t * __init fixmap_pmd(unsigned long addr) { - pgd_t *pgd = pgd_offset_k(addr); - pud_t *pud = pud_offset(pgd, addr); - pmd_t *pmd = pmd_offset(pud, addr); - - return pmd; + return pmd_off_k(addr); } void __init early_fixmap_init(void) @@ -409,13 +400,12 @@ void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) pte_t *pte = pte_offset_fixmap(pmd_off_k(vaddr), vaddr); /* Make sure fixmap region does not exceed available allocation. */ - BUILD_BUG_ON(FIXADDR_START + (__end_of_fixed_addresses * PAGE_SIZE) > - FIXADDR_END); + BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) < FIXADDR_START); BUG_ON(idx >= __end_of_fixed_addresses); - /* we only support device mappings until pgprot_kernel has been set */ + /* We support only device mappings before pgprot_kernel is set. */ if (WARN_ON(pgprot_val(prot) != pgprot_val(FIXMAP_PAGE_IO) && - pgprot_val(pgprot_kernel) == 0)) + pgprot_val(prot) && pgprot_val(pgprot_kernel) == 0)) return; if (pgprot_val(prot)) @@ -426,6 +416,26 @@ void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) local_flush_tlb_kernel_range(vaddr, vaddr + PAGE_SIZE); } +static pgprot_t protection_map[16] __ro_after_init = { + [VM_NONE] = __PAGE_NONE, + [VM_READ] = __PAGE_READONLY, + [VM_WRITE] = __PAGE_COPY, + [VM_WRITE | VM_READ] = __PAGE_COPY, + [VM_EXEC] = __PAGE_READONLY_EXEC, + [VM_EXEC | VM_READ] = __PAGE_READONLY_EXEC, + [VM_EXEC | VM_WRITE] = __PAGE_COPY_EXEC, + [VM_EXEC | VM_WRITE | VM_READ] = __PAGE_COPY_EXEC, + [VM_SHARED] = __PAGE_NONE, + [VM_SHARED | VM_READ] = __PAGE_READONLY, + [VM_SHARED | VM_WRITE] = __PAGE_SHARED, + [VM_SHARED | VM_WRITE | VM_READ] = __PAGE_SHARED, + [VM_SHARED | VM_EXEC] = __PAGE_READONLY_EXEC, + [VM_SHARED | VM_EXEC | VM_READ] = __PAGE_READONLY_EXEC, + [VM_SHARED | VM_EXEC | VM_WRITE] = __PAGE_SHARED_EXEC, + [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = __PAGE_SHARED_EXEC +}; +DECLARE_VM_GET_PAGE_PROT + /* * Adjust the PMD section entries according to the CPU in use. */ @@ -434,7 +444,6 @@ static void __init build_mem_type_table(void) struct cachepolicy *cp; unsigned int cr = get_cr(); pteval_t user_pgprot, kern_pgprot, vecs_pgprot; - pteval_t hyp_device_pgprot, s2_pgprot, s2_device_pgprot; int cpu_arch = cpu_architecture(); int i; @@ -511,6 +520,7 @@ static void __init build_mem_type_table(void) /* Also setup NX memory mapping */ mem_types[MT_MEMORY_RW].prot_sect |= PMD_SECT_XN; + mem_types[MT_MEMORY_RO].prot_sect |= PMD_SECT_XN; } if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) { /* @@ -558,9 +568,6 @@ static void __init build_mem_type_table(void) */ cp = &cache_policies[cachepolicy]; vecs_pgprot = kern_pgprot = user_pgprot = cp->pte; - s2_pgprot = cp->pte_s2; - hyp_device_pgprot = mem_types[MT_DEVICE].prot_pte; - s2_device_pgprot = mem_types[MT_DEVICE].prot_pte_s2; #ifndef CONFIG_ARM_LPAE /* @@ -593,6 +600,7 @@ static void __init build_mem_type_table(void) mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; + mem_types[MT_MEMORY_RO].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; #endif /* @@ -604,7 +612,6 @@ static void __init build_mem_type_table(void) user_pgprot |= L_PTE_SHARED; kern_pgprot |= L_PTE_SHARED; vecs_pgprot |= L_PTE_SHARED; - s2_pgprot |= L_PTE_SHARED; mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S; mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED; mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S; @@ -613,6 +620,8 @@ static void __init build_mem_type_table(void) mem_types[MT_MEMORY_RWX].prot_pte |= L_PTE_SHARED; mem_types[MT_MEMORY_RW].prot_sect |= PMD_SECT_S; mem_types[MT_MEMORY_RW].prot_pte |= L_PTE_SHARED; + mem_types[MT_MEMORY_RO].prot_sect |= PMD_SECT_S; + mem_types[MT_MEMORY_RO].prot_pte |= L_PTE_SHARED; mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED; mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= PMD_SECT_S; mem_types[MT_MEMORY_RWX_NONCACHED].prot_pte |= L_PTE_SHARED; @@ -666,9 +675,6 @@ static void __init build_mem_type_table(void) pgprot_user = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot); pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | kern_pgprot); - pgprot_s2 = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | s2_pgprot); - pgprot_s2_device = __pgprot(s2_device_pgprot); - pgprot_hyp_device = __pgprot(hyp_device_pgprot); mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask; mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask; @@ -676,6 +682,8 @@ static void __init build_mem_type_table(void) mem_types[MT_MEMORY_RWX].prot_pte |= kern_pgprot; mem_types[MT_MEMORY_RW].prot_sect |= ecc_mask | cp->pmd; mem_types[MT_MEMORY_RW].prot_pte |= kern_pgprot; + mem_types[MT_MEMORY_RO].prot_sect |= ecc_mask | cp->pmd; + mem_types[MT_MEMORY_RO].prot_pte |= kern_pgprot; mem_types[MT_MEMORY_DMA_READY].prot_pte |= kern_pgprot; mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= ecc_mask; mem_types[MT_ROM].prot_sect |= cp->pmd; @@ -718,22 +726,18 @@ EXPORT_SYMBOL(phys_mem_access_prot); static void __init *early_alloc(unsigned long sz) { - void *ptr = memblock_alloc(sz, sz); - - if (!ptr) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", - __func__, sz, sz); + return memblock_alloc_or_panic(sz, sz); - return ptr; } static void *__init late_alloc(unsigned long sz) { - void *ptr = (void *)__get_free_pages(GFP_PGTABLE_KERNEL, get_order(sz)); + void *ptdesc = pagetable_alloc(GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM, + get_order(sz)); - if (!ptr || !pgtable_pte_page_ctor(virt_to_page(ptr))) + if (!ptdesc || !pagetable_pte_ctor(ptdesc)) BUG(); - return ptr; + return ptdesc_to_virt(ptdesc); } static pte_t * __init arm_pte_alloc(pmd_t *pmd, unsigned long addr, @@ -827,12 +831,12 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr, } while (pmd++, addr = next, addr != end); } -static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, +static void __init alloc_init_pud(p4d_t *p4d, unsigned long addr, unsigned long end, phys_addr_t phys, const struct mem_type *type, void *(*alloc)(unsigned long sz), bool ng) { - pud_t *pud = pud_offset(pgd, addr); + pud_t *pud = pud_offset(p4d, addr); unsigned long next; do { @@ -842,6 +846,21 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, } while (pud++, addr = next, addr != end); } +static void __init alloc_init_p4d(pgd_t *pgd, unsigned long addr, + unsigned long end, phys_addr_t phys, + const struct mem_type *type, + void *(*alloc)(unsigned long sz), bool ng) +{ + p4d_t *p4d = p4d_offset(pgd, addr); + unsigned long next; + + do { + next = p4d_addr_end(addr, end); + alloc_init_pud(p4d, addr, next, phys, type, alloc, ng); + phys += next - addr; + } while (p4d++, addr = next, addr != end); +} + #ifndef CONFIG_ARM_LPAE static void __init create_36bit_mapping(struct mm_struct *mm, struct map_desc *md, @@ -889,7 +908,8 @@ static void __init create_36bit_mapping(struct mm_struct *mm, pgd = pgd_offset(mm, addr); end = addr + length; do { - pud_t *pud = pud_offset(pgd, addr); + p4d_t *p4d = p4d_offset(pgd, addr); + pud_t *pud = pud_offset(p4d, addr); pmd_t *pmd = pmd_offset(pud, addr); int i; @@ -940,7 +960,7 @@ static void __init __create_mapping(struct mm_struct *mm, struct map_desc *md, do { unsigned long next = pgd_addr_end(addr, end); - alloc_init_pud(pgd, addr, next, phys, type, alloc, ng); + alloc_init_p4d(pgd, addr, next, phys, type, alloc, ng); phys += next - addr; addr = next; @@ -962,7 +982,7 @@ static void __init create_mapping(struct map_desc *md) return; } - if ((md->type == MT_DEVICE || md->type == MT_ROM) && + if (md->type == MT_DEVICE && md->virtual >= PAGE_OFFSET && md->virtual < FIXADDR_START && (md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) { pr_warn("BUG: mapping for 0x%08llx at 0x%08lx out of vmalloc space\n", @@ -976,7 +996,13 @@ void __init create_mapping_late(struct mm_struct *mm, struct map_desc *md, bool ng) { #ifdef CONFIG_ARM_LPAE - pud_t *pud = pud_alloc(mm, pgd_offset(mm, md->virtual), md->virtual); + p4d_t *p4d; + pud_t *pud; + + p4d = p4d_alloc(mm, pgd_offset(mm, md->virtual), md->virtual); + if (WARN_ON(!p4d)) + return; + pud = pud_alloc(mm, p4d, md->virtual); if (WARN_ON(!pud)) return; pmd_alloc(mm, pud, 0); @@ -996,10 +1022,7 @@ void __init iotable_init(struct map_desc *io_desc, int nr) if (!nr) return; - svm = memblock_alloc(sizeof(*svm) * nr, __alignof__(*svm)); - if (!svm) - panic("%s: Failed to allocate %zu bytes align=0x%zx\n", - __func__, sizeof(*svm) * nr, __alignof__(*svm)); + svm = memblock_alloc_or_panic(sizeof(*svm) * nr, __alignof__(*svm)); for (md = io_desc; nr; md++, nr--) { create_mapping(md); @@ -1021,10 +1044,7 @@ void __init vm_reserve_area_early(unsigned long addr, unsigned long size, struct vm_struct *vm; struct static_vm *svm; - svm = memblock_alloc(sizeof(*svm), __alignof__(*svm)); - if (!svm) - panic("%s: Failed to allocate %zu bytes align=0x%zx\n", - __func__, sizeof(*svm), __alignof__(*svm)); + svm = memblock_alloc_or_panic(sizeof(*svm), __alignof__(*svm)); vm = &svm->vm; vm->addr = (void *)addr; @@ -1130,31 +1150,32 @@ void __init debug_ll_io_init(void) } #endif -static void * __initdata vmalloc_min = - (void *)(VMALLOC_END - (240 << 20) - VMALLOC_OFFSET); +static unsigned long __initdata vmalloc_size = 240 * SZ_1M; /* * vmalloc=size forces the vmalloc area to be exactly 'size' * bytes. This can be used to increase (or decrease) the vmalloc - * area - the default is 240m. + * area - the default is 240MiB. */ static int __init early_vmalloc(char *arg) { unsigned long vmalloc_reserve = memparse(arg, NULL); + unsigned long vmalloc_max; if (vmalloc_reserve < SZ_16M) { vmalloc_reserve = SZ_16M; - pr_warn("vmalloc area too small, limiting to %luMB\n", + pr_warn("vmalloc area is too small, limiting to %luMiB\n", vmalloc_reserve >> 20); } - if (vmalloc_reserve > VMALLOC_END - (PAGE_OFFSET + SZ_32M)) { - vmalloc_reserve = VMALLOC_END - (PAGE_OFFSET + SZ_32M); - pr_warn("vmalloc area is too big, limiting to %luMB\n", + vmalloc_max = VMALLOC_END - (PAGE_OFFSET + SZ_32M + VMALLOC_OFFSET); + if (vmalloc_reserve > vmalloc_max) { + vmalloc_reserve = vmalloc_max; + pr_warn("vmalloc area is too big, limiting to %luMiB\n", vmalloc_reserve >> 20); } - vmalloc_min = (void *)(VMALLOC_END - vmalloc_reserve); + vmalloc_size = vmalloc_reserve; return 0; } early_param("vmalloc", early_vmalloc); @@ -1163,9 +1184,8 @@ phys_addr_t arm_lowmem_limit __initdata = 0; void __init adjust_lowmem_bounds(void) { - phys_addr_t memblock_limit = 0; - u64 vmalloc_limit; - struct memblock_region *reg; + phys_addr_t block_start, block_end, memblock_limit = 0; + u64 vmalloc_limit, i; phys_addr_t lowmem_limit = 0; /* @@ -1175,32 +1195,25 @@ void __init adjust_lowmem_bounds(void) * and may itself be outside the valid range for which phys_addr_t * and therefore __pa() is defined. */ - vmalloc_limit = (u64)(uintptr_t)vmalloc_min - PAGE_OFFSET + PHYS_OFFSET; + vmalloc_limit = (u64)VMALLOC_END - vmalloc_size - VMALLOC_OFFSET - + PAGE_OFFSET + PHYS_OFFSET; /* * The first usable region must be PMD aligned. Mark its start * as MEMBLOCK_NOMAP if it isn't */ - for_each_memblock(memory, reg) { - if (!memblock_is_nomap(reg)) { - if (!IS_ALIGNED(reg->base, PMD_SIZE)) { - phys_addr_t len; + for_each_mem_range(i, &block_start, &block_end) { + if (!IS_ALIGNED(block_start, PMD_SIZE)) { + phys_addr_t len; - len = round_up(reg->base, PMD_SIZE) - reg->base; - memblock_mark_nomap(reg->base, len); - } - break; + len = round_up(block_start, PMD_SIZE) - block_start; + memblock_mark_nomap(block_start, len); } + break; } - for_each_memblock(memory, reg) { - phys_addr_t block_start = reg->base; - phys_addr_t block_end = reg->base + reg->size; - - if (memblock_is_nomap(reg)) - continue; - - if (reg->base < vmalloc_limit) { + for_each_mem_range(i, &block_start, &block_end) { + if (block_start < vmalloc_limit) { if (block_end > lowmem_limit) /* * Compare as u64 to ensure vmalloc_limit does @@ -1264,7 +1277,7 @@ void __init adjust_lowmem_bounds(void) memblock_set_current_limit(memblock_limit); } -static inline void prepare_page_table(void) +static __init void prepare_page_table(void) { unsigned long addr; phys_addr_t end; @@ -1272,8 +1285,25 @@ static inline void prepare_page_table(void) /* * Clear out all the mappings below the kernel image. */ +#ifdef CONFIG_KASAN + /* + * KASan's shadow memory inserts itself between the TASK_SIZE + * and MODULES_VADDR. Do not clear the KASan shadow memory mappings. + */ + for (addr = 0; addr < KASAN_SHADOW_START; addr += PMD_SIZE) + pmd_clear(pmd_off_k(addr)); + /* + * Skip over the KASan shadow area. KASAN_SHADOW_END is sometimes + * equal to MODULES_VADDR and then we exit the pmd clearing. If we + * are using a thumb-compiled kernel, there there will be 8MB more + * to clear as KASan always offset to 16 MB below MODULES_VADDR. + */ + for (addr = KASAN_SHADOW_END; addr < MODULES_VADDR; addr += PMD_SIZE) + pmd_clear(pmd_off_k(addr)); +#else for (addr = 0; addr < MODULES_VADDR; addr += PMD_SIZE) pmd_clear(pmd_off_k(addr)); +#endif #ifdef CONFIG_XIP_KERNEL /* The XIP kernel is mapped in the module area -- skip over it */ @@ -1352,17 +1382,14 @@ static void __init devicemaps_init(const struct machine_desc *mdesc) for (addr = VMALLOC_START; addr < (FIXADDR_TOP & PMD_MASK); addr += PMD_SIZE) pmd_clear(pmd_off_k(addr)); - /* - * Map the kernel if it is XIP. - * It is always first in the modulearea. - */ -#ifdef CONFIG_XIP_KERNEL - map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK); - map.virtual = MODULES_VADDR; - map.length = ((unsigned long)_exiprom - map.virtual + ~SECTION_MASK) & SECTION_MASK; - map.type = MT_ROM; - create_mapping(&map); -#endif + if (__atags_pointer) { + /* create a read-only mapping of the device tree */ + map.pfn = __phys_to_pfn(__atags_pointer & SECTION_MASK); + map.virtual = FDT_FIXED_BASE; + map.length = FDT_FIXED_SIZE; + map.type = MT_MEMORY_RO; + create_mapping(&map); + } /* * Map the cache flushing regions. @@ -1449,71 +1476,150 @@ static void __init kmap_init(void) static void __init map_lowmem(void) { - struct memblock_region *reg; - phys_addr_t kernel_x_start = round_down(__pa(KERNEL_START), SECTION_SIZE); - phys_addr_t kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE); + phys_addr_t start, end; + u64 i; /* Map all the lowmem memory banks. */ - for_each_memblock(memory, reg) { - phys_addr_t start = reg->base; - phys_addr_t end = start + reg->size; + for_each_mem_range(i, &start, &end) { struct map_desc map; - if (memblock_is_nomap(reg)) - continue; - + pr_debug("map lowmem start: 0x%08llx, end: 0x%08llx\n", + (long long)start, (long long)end); if (end > arm_lowmem_limit) end = arm_lowmem_limit; if (start >= end) break; - if (end < kernel_x_start) { - map.pfn = __phys_to_pfn(start); - map.virtual = __phys_to_virt(start); - map.length = end - start; - map.type = MT_MEMORY_RWX; + /* + * If our kernel image is in the VMALLOC area we need to remove + * the kernel physical memory from lowmem since the kernel will + * be mapped separately. + * + * The kernel will typically be at the very start of lowmem, + * but any placement relative to memory ranges is possible. + * + * If the memblock contains the kernel, we have to chisel out + * the kernel memory from it and map each part separately. We + * get 6 different theoretical cases: + * + * +--------+ +--------+ + * +-- start --+ +--------+ | Kernel | | Kernel | + * | | | Kernel | | case 2 | | case 5 | + * | | | case 1 | +--------+ | | +--------+ + * | Memory | +--------+ | | | Kernel | + * | range | +--------+ | | | case 6 | + * | | | Kernel | +--------+ | | +--------+ + * | | | case 3 | | Kernel | | | + * +-- end ----+ +--------+ | case 4 | | | + * +--------+ +--------+ + */ - create_mapping(&map); - } else if (start >= kernel_x_end) { - map.pfn = __phys_to_pfn(start); - map.virtual = __phys_to_virt(start); - map.length = end - start; - map.type = MT_MEMORY_RW; + /* Case 5: kernel covers range, don't map anything, should be rare */ + if ((start > kernel_sec_start) && (end < kernel_sec_end)) + break; - create_mapping(&map); - } else { - /* This better cover the entire kernel */ - if (start < kernel_x_start) { + /* Cases where the kernel is starting inside the range */ + if ((kernel_sec_start >= start) && (kernel_sec_start <= end)) { + /* Case 6: kernel is embedded in the range, we need two mappings */ + if ((start < kernel_sec_start) && (end > kernel_sec_end)) { + /* Map memory below the kernel */ map.pfn = __phys_to_pfn(start); map.virtual = __phys_to_virt(start); - map.length = kernel_x_start - start; + map.length = kernel_sec_start - start; map.type = MT_MEMORY_RW; - create_mapping(&map); - } - - map.pfn = __phys_to_pfn(kernel_x_start); - map.virtual = __phys_to_virt(kernel_x_start); - map.length = kernel_x_end - kernel_x_start; - map.type = MT_MEMORY_RWX; - - create_mapping(&map); - - if (kernel_x_end < end) { - map.pfn = __phys_to_pfn(kernel_x_end); - map.virtual = __phys_to_virt(kernel_x_end); - map.length = end - kernel_x_end; + /* Map memory above the kernel */ + map.pfn = __phys_to_pfn(kernel_sec_end); + map.virtual = __phys_to_virt(kernel_sec_end); + map.length = end - kernel_sec_end; map.type = MT_MEMORY_RW; - create_mapping(&map); + break; } + /* Case 1: kernel and range start at the same address, should be common */ + if (kernel_sec_start == start) + start = kernel_sec_end; + /* Case 3: kernel and range end at the same address, should be rare */ + if (kernel_sec_end == end) + end = kernel_sec_start; + } else if ((kernel_sec_start < start) && (kernel_sec_end > start) && (kernel_sec_end < end)) { + /* Case 2: kernel ends inside range, starts below it */ + start = kernel_sec_end; + } else if ((kernel_sec_start > start) && (kernel_sec_start < end) && (kernel_sec_end > end)) { + /* Case 4: kernel starts inside range, ends above it */ + end = kernel_sec_start; } + map.pfn = __phys_to_pfn(start); + map.virtual = __phys_to_virt(start); + map.length = end - start; + map.type = MT_MEMORY_RW; + create_mapping(&map); } } +static void __init map_kernel(void) +{ + /* + * We use the well known kernel section start and end and split the area in the + * middle like this: + * . . + * | RW memory | + * +----------------+ kernel_x_start + * | Executable | + * | kernel memory | + * +----------------+ kernel_x_end / kernel_nx_start + * | Non-executable | + * | kernel memory | + * +----------------+ kernel_nx_end + * | RW memory | + * . . + * + * Notice that we are dealing with section sized mappings here so all of this + * will be bumped to the closest section boundary. This means that some of the + * non-executable part of the kernel memory is actually mapped as executable. + * This will only persist until we turn on proper memory management later on + * and we remap the whole kernel with page granularity. + */ +#ifdef CONFIG_XIP_KERNEL + phys_addr_t kernel_nx_start = kernel_sec_start; +#else + phys_addr_t kernel_x_start = kernel_sec_start; + phys_addr_t kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE); + phys_addr_t kernel_nx_start = kernel_x_end; +#endif + phys_addr_t kernel_nx_end = kernel_sec_end; + struct map_desc map; + + /* + * Map the kernel if it is XIP. + * It is always first in the modulearea. + */ +#ifdef CONFIG_XIP_KERNEL + map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK); + map.virtual = MODULES_VADDR; + map.length = ((unsigned long)_exiprom - map.virtual + ~SECTION_MASK) & SECTION_MASK; + map.type = MT_ROM; + create_mapping(&map); +#else + map.pfn = __phys_to_pfn(kernel_x_start); + map.virtual = __phys_to_virt(kernel_x_start); + map.length = kernel_x_end - kernel_x_start; + map.type = MT_MEMORY_RWX; + create_mapping(&map); + + /* If the nx part is small it may end up covered by the tail of the RWX section */ + if (kernel_x_end == kernel_nx_end) + return; +#endif + map.pfn = __phys_to_pfn(kernel_nx_start); + map.virtual = __phys_to_virt(kernel_nx_start); + map.length = kernel_nx_end - kernel_nx_start; + map.type = MT_MEMORY_RW; + create_mapping(&map); +} + #ifdef CONFIG_ARM_PV_FIXUP -extern unsigned long __atags_pointer; -typedef void pgtables_remap(long long offset, unsigned long pgd, void *bdata); +typedef void pgtables_remap(long long offset, unsigned long pgd); pgtables_remap lpae_pgtables_remap_asm; /* @@ -1524,9 +1630,8 @@ static void __init early_paging_init(const struct machine_desc *mdesc) { pgtables_remap *lpae_pgtables_remap; unsigned long pa_pgd; - unsigned int cr, ttbcr; + u32 cr, ttbcr, tmp; long long offset; - void *boot_data; if (!mdesc->pv_fixup) return; @@ -1536,6 +1641,13 @@ static void __init early_paging_init(const struct machine_desc *mdesc) return; /* + * Offset the kernel section physical offsets so that the kernel + * mapping will work out later on. + */ + kernel_sec_start += offset; + kernel_sec_end += offset; + + /* * Get the address of the remap function in the 1:1 identity * mapping setup by the early page table assembly code. We * must get this prior to the pv update. The following barrier @@ -1543,7 +1655,6 @@ static void __init early_paging_init(const struct machine_desc *mdesc) */ lpae_pgtables_remap = (pgtables_remap *)(unsigned long)__pa(lpae_pgtables_remap_asm); pa_pgd = __pa(swapper_pg_dir); - boot_data = __va(__atags_pointer); barrier(); pr_info("Switching physical address space to 0x%08llx\n", @@ -1568,9 +1679,10 @@ static void __init early_paging_init(const struct machine_desc *mdesc) */ cr = get_cr(); set_cr(cr & ~(CR_I | CR_C)); - asm("mrc p15, 0, %0, c2, c0, 2" : "=r" (ttbcr)); - asm volatile("mcr p15, 0, %0, c2, c0, 2" - : : "r" (ttbcr & ~(3 << 8 | 3 << 10))); + ttbcr = cpu_get_ttbcr(); + /* Disable all kind of caching of the translation table */ + tmp = ttbcr & ~(TTBCR_ORGN0_MASK | TTBCR_IRGN0_MASK); + cpu_set_ttbcr(tmp); flush_cache_all(); /* @@ -1579,10 +1691,10 @@ static void __init early_paging_init(const struct machine_desc *mdesc) * needs to be assembly. It's fairly simple, as we're using the * temporary tables setup by the initial assembly code. */ - lpae_pgtables_remap(offset, pa_pgd, boot_data); + lpae_pgtables_remap(offset, pa_pgd); /* Re-enable the caches and cacheable TLB walks */ - asm volatile("mcr p15, 0, %0, c2, c0, 2" : : "r" (ttbcr)); + cpu_set_ttbcr(ttbcr); set_cr(cr); } @@ -1644,9 +1756,23 @@ void __init paging_init(const struct machine_desc *mdesc) { void *zero_page; +#ifdef CONFIG_XIP_KERNEL + /* Store the kernel RW RAM region start/end in these variables */ + kernel_sec_start = CONFIG_PHYS_OFFSET & SECTION_MASK; + kernel_sec_end = round_up(__pa(_end), SECTION_SIZE); +#endif + pr_debug("physical kernel sections: 0x%08llx-0x%08llx\n", + kernel_sec_start, kernel_sec_end); + prepare_page_table(); map_lowmem(); memblock_set_current_limit(arm_lowmem_limit); + pr_debug("lowmem limit is %08llx\n", (long long)arm_lowmem_limit); + /* + * After this point early_alloc(), i.e. the memblock allocator, can + * be used + */ + map_kernel(); dma_contiguous_remap(); early_fixmap_shutdown(); devicemaps_init(mdesc); @@ -1661,10 +1787,7 @@ void __init paging_init(const struct machine_desc *mdesc) bootmem_init(); empty_zero_page = virt_to_page(zero_page); - __flush_dcache_page(NULL, empty_zero_page); - - /* Compute the virt/idmap offset, mostly for the sake of KVM */ - kimage_voffset = (unsigned long)&kimage_voffset - virt_to_idmap(&kimage_voffset); + __flush_dcache_folio(NULL, page_folio(empty_zero_page)); } void __init early_mm_init(const struct machine_desc *mdesc) @@ -1672,3 +1795,23 @@ void __init early_mm_init(const struct machine_desc *mdesc) build_mem_type_table(); early_paging_init(mdesc); } + +void set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pteval, unsigned int nr) +{ + unsigned long ext = 0; + + if (addr < TASK_SIZE && pte_valid_user(pteval)) { + if (!pte_special(pteval)) + __sync_icache_dcache(pteval); + ext |= PTE_EXT_NG; + } + + for (;;) { + set_pte_ext(ptep, pteval, ext); + if (--nr == 0) + break; + ptep++; + pteval = pte_next_pfn(pteval); + } +} diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c index 8b3d7191e2b8..d638cc87807e 100644 --- a/arch/arm/mm/nommu.c +++ b/arch/arm/mm/nommu.c @@ -21,11 +21,19 @@ #include <asm/cputype.h> #include <asm/mpu.h> #include <asm/procinfo.h> +#include <asm/idmap.h> #include "mm.h" unsigned long vectors_base; +/* + * empty_zero_page is a special page that is used for + * zero-initialized data and COW. + */ +struct page *empty_zero_page; +EXPORT_SYMBOL(empty_zero_page); + #ifdef CONFIG_ARM_MPU struct mpu_rgn_info mpu_rgn_info; #endif @@ -148,9 +156,18 @@ void __init adjust_lowmem_bounds(void) */ void __init paging_init(const struct machine_desc *mdesc) { + void *zero_page; + early_trap_init((void *)vectors_base); mpu_setup(); + + /* allocate the zero page. */ + zero_page = (void *)memblock_alloc_or_panic(PAGE_SIZE, PAGE_SIZE); + bootmem_init(); + + empty_zero_page = virt_to_page(zero_page); + flush_dcache_page(empty_zero_page); } /* @@ -160,17 +177,17 @@ void setup_mm_for_reboot(void) { } -void flush_dcache_page(struct page *page) +void flush_dcache_folio(struct folio *folio) { - __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE); + __cpuc_flush_dcache_area(folio_address(folio), folio_size(folio)); } -EXPORT_SYMBOL(flush_dcache_page); +EXPORT_SYMBOL(flush_dcache_folio); -void flush_kernel_dcache_page(struct page *page) +void flush_dcache_page(struct page *page) { __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE); } -EXPORT_SYMBOL(flush_kernel_dcache_page); +EXPORT_SYMBOL(flush_dcache_page); void copy_to_user_page(struct vm_area_struct *vma, struct page *page, unsigned long uaddr, void *dst, const void *src, @@ -231,19 +248,12 @@ void __iomem *pci_remap_cfgspace(resource_size_t res_cookie, size_t size) EXPORT_SYMBOL_GPL(pci_remap_cfgspace); #endif -void *arch_memremap_wb(phys_addr_t phys_addr, size_t size) +void *arch_memremap_wb(phys_addr_t phys_addr, size_t size, unsigned long flags) { return (void *)phys_addr; } -void __iounmap(volatile void __iomem *addr) -{ -} -EXPORT_SYMBOL(__iounmap); - -void (*arch_iounmap)(volatile void __iomem *); - -void iounmap(volatile void __iomem *addr) +void iounmap(volatile void __iomem *io_addr) { } EXPORT_SYMBOL(iounmap); diff --git a/arch/arm/mm/pageattr.c b/arch/arm/mm/pageattr.c index d546efad7e97..064ad508c149 100644 --- a/arch/arm/mm/pageattr.c +++ b/arch/arm/mm/pageattr.c @@ -5,7 +5,6 @@ #include <linux/mm.h> #include <linux/module.h> -#include <asm/pgtable.h> #include <asm/tlbflush.h> #include <asm/set_memory.h> @@ -26,39 +25,49 @@ static int change_page_range(pte_t *ptep, unsigned long addr, void *data) return 0; } -static bool in_range(unsigned long start, unsigned long size, +static bool range_in_range(unsigned long start, unsigned long size, unsigned long range_start, unsigned long range_end) { return start >= range_start && start < range_end && size <= range_end - start; } +/* + * This function assumes that the range is mapped with PAGE_SIZE pages. + */ +static int __change_memory_common(unsigned long start, unsigned long size, + pgprot_t set_mask, pgprot_t clear_mask) +{ + struct page_change_data data; + int ret; + + data.set_mask = set_mask; + data.clear_mask = clear_mask; + + ret = apply_to_page_range(&init_mm, start, size, change_page_range, + &data); + + flush_tlb_kernel_range(start, start + size); + return ret; +} + static int change_memory_common(unsigned long addr, int numpages, pgprot_t set_mask, pgprot_t clear_mask) { unsigned long start = addr & PAGE_MASK; unsigned long end = PAGE_ALIGN(addr) + numpages * PAGE_SIZE; unsigned long size = end - start; - int ret; - struct page_change_data data; WARN_ON_ONCE(start != addr); if (!size) return 0; - if (!in_range(start, size, MODULES_VADDR, MODULES_END) && - !in_range(start, size, VMALLOC_START, VMALLOC_END)) + if (!range_in_range(start, size, MODULES_VADDR, MODULES_END) && + !range_in_range(start, size, VMALLOC_START, VMALLOC_END)) return -EINVAL; - data.set_mask = set_mask; - data.clear_mask = clear_mask; - - ret = apply_to_page_range(&init_mm, start, size, change_page_range, - &data); - - flush_tlb_kernel_range(start, end); - return ret; + return __change_memory_common(start, size, set_mask, clear_mask); } int set_memory_ro(unsigned long addr, int numpages) @@ -88,3 +97,15 @@ int set_memory_x(unsigned long addr, int numpages) __pgprot(0), __pgprot(L_PTE_XN)); } + +int set_memory_valid(unsigned long addr, int numpages, int enable) +{ + if (enable) + return __change_memory_common(addr, PAGE_SIZE * numpages, + __pgprot(L_PTE_VALID), + __pgprot(0)); + else + return __change_memory_common(addr, PAGE_SIZE * numpages, + __pgprot(0), + __pgprot(L_PTE_VALID)); +} diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c index 478bd2c6aa50..4eb81b7ed03a 100644 --- a/arch/arm/mm/pgd.c +++ b/arch/arm/mm/pgd.c @@ -17,11 +17,11 @@ #include "mm.h" #ifdef CONFIG_ARM_LPAE -#define __pgd_alloc() kmalloc_array(PTRS_PER_PGD, sizeof(pgd_t), GFP_KERNEL) -#define __pgd_free(pgd) kfree(pgd) +#define _pgd_alloc(mm) kmalloc_array(PTRS_PER_PGD, sizeof(pgd_t), GFP_KERNEL | __GFP_ZERO) +#define _pgd_free(mm, pgd) kfree(pgd) #else -#define __pgd_alloc() (pgd_t *)__get_free_pages(GFP_KERNEL, 2) -#define __pgd_free(pgd) free_pages((unsigned long)pgd, 2) +#define _pgd_alloc(mm) __pgd_alloc(mm, 2) +#define _pgd_free(mm, pgd) __pgd_free(mm, pgd) #endif /* @@ -30,16 +30,15 @@ pgd_t *pgd_alloc(struct mm_struct *mm) { pgd_t *new_pgd, *init_pgd; + p4d_t *new_p4d, *init_p4d; pud_t *new_pud, *init_pud; pmd_t *new_pmd, *init_pmd; pte_t *new_pte, *init_pte; - new_pgd = __pgd_alloc(); + new_pgd = _pgd_alloc(mm); if (!new_pgd) goto no_pgd; - memset(new_pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); - /* * Copy over the kernel and IO PGD entries */ @@ -53,15 +52,33 @@ pgd_t *pgd_alloc(struct mm_struct *mm) /* * Allocate PMD table for modules and pkmap mappings. */ - new_pud = pud_alloc(mm, new_pgd + pgd_index(MODULES_VADDR), + new_p4d = p4d_alloc(mm, new_pgd + pgd_index(MODULES_VADDR), MODULES_VADDR); + if (!new_p4d) + goto no_p4d; + + new_pud = pud_alloc(mm, new_p4d, MODULES_VADDR); if (!new_pud) goto no_pud; new_pmd = pmd_alloc(mm, new_pud, 0); if (!new_pmd) goto no_pmd; -#endif +#ifdef CONFIG_KASAN + /* + * Copy PMD table for KASAN shadow mappings. + */ + init_pgd = pgd_offset_k(TASK_SIZE); + init_p4d = p4d_offset(init_pgd, TASK_SIZE); + init_pud = pud_offset(init_p4d, TASK_SIZE); + init_pmd = pmd_offset(init_pud, TASK_SIZE); + new_pmd = pmd_offset(new_pud, TASK_SIZE); + memcpy(new_pmd, init_pmd, + (pmd_index(MODULES_VADDR) - pmd_index(TASK_SIZE)) + * sizeof(pmd_t)); + clean_dcache_area(new_pmd, PTRS_PER_PMD * sizeof(pmd_t)); +#endif /* CONFIG_KASAN */ +#endif /* CONFIG_LPAE */ if (!vectors_high()) { /* @@ -69,7 +86,11 @@ pgd_t *pgd_alloc(struct mm_struct *mm) * contains the machine vectors. The vectors are always high * with LPAE. */ - new_pud = pud_alloc(mm, new_pgd, 0); + new_p4d = p4d_alloc(mm, new_pgd, 0); + if (!new_p4d) + goto no_p4d; + + new_pud = pud_alloc(mm, new_p4d, 0); if (!new_pud) goto no_pud; @@ -91,7 +112,8 @@ pgd_t *pgd_alloc(struct mm_struct *mm) pmd_val(*new_pmd) |= PMD_DOMAIN(DOMAIN_VECTORS); #endif - init_pud = pud_offset(init_pgd, 0); + init_p4d = p4d_offset(init_pgd, 0); + init_pud = pud_offset(init_p4d, 0); init_pmd = pmd_offset(init_pud, 0); init_pte = pte_offset_map(init_pmd, 0); set_pte_ext(new_pte + 0, init_pte[0], 0); @@ -108,7 +130,9 @@ no_pte: no_pmd: pud_free(mm, new_pud); no_pud: - __pgd_free(new_pgd); + p4d_free(mm, new_p4d); +no_p4d: + _pgd_free(mm, new_pgd); no_pgd: return NULL; } @@ -116,6 +140,7 @@ no_pgd: void pgd_free(struct mm_struct *mm, pgd_t *pgd_base) { pgd_t *pgd; + p4d_t *p4d; pud_t *pud; pmd_t *pmd; pgtable_t pte; @@ -127,7 +152,11 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base) if (pgd_none_or_clear_bad(pgd)) goto no_pgd; - pud = pud_offset(pgd, 0); + p4d = p4d_offset(pgd, 0); + if (p4d_none_or_clear_bad(p4d)) + goto no_p4d; + + pud = pud_offset(p4d, 0); if (pud_none_or_clear_bad(pud)) goto no_pud; @@ -144,8 +173,11 @@ no_pmd: pmd_free(mm, pmd); mm_dec_nr_pmds(mm); no_pud: - pgd_clear(pgd); + p4d_clear(p4d); pud_free(mm, pud); +no_p4d: + pgd_clear(pgd); + p4d_free(mm, p4d); no_pgd: #ifdef CONFIG_ARM_LPAE /* @@ -156,16 +188,22 @@ no_pgd: continue; if (pgd_val(*pgd) & L_PGD_SWAPPER) continue; - pud = pud_offset(pgd, 0); + p4d = p4d_offset(pgd, 0); + if (p4d_none_or_clear_bad(p4d)) + continue; + pud = pud_offset(p4d, 0); if (pud_none_or_clear_bad(pud)) continue; pmd = pmd_offset(pud, 0); pud_clear(pud); pmd_free(mm, pmd); mm_dec_nr_pmds(mm); - pgd_clear(pgd); + p4d_clear(p4d); pud_free(mm, pud); + mm_dec_nr_puds(mm); + pgd_clear(pgd); + p4d_free(mm, p4d); } #endif - __pgd_free(pgd_base); + _pgd_free(mm, pgd_base); } diff --git a/arch/arm/mm/physaddr.c b/arch/arm/mm/physaddr.c index cf75819e4c13..3f263c840ebc 100644 --- a/arch/arm/mm/physaddr.c +++ b/arch/arm/mm/physaddr.c @@ -6,7 +6,7 @@ #include <linux/mm.h> #include <asm/sections.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/fixmap.h> #include <asm/dma.h> diff --git a/arch/arm/mm/pmsa-v7.c b/arch/arm/mm/pmsa-v7.c index 699fa2e88725..59d916ccdf25 100644 --- a/arch/arm/mm/pmsa-v7.c +++ b/arch/arm/mm/pmsa-v7.c @@ -231,12 +231,13 @@ static int __init allocate_region(phys_addr_t base, phys_addr_t size, void __init pmsav7_adjust_lowmem_bounds(void) { phys_addr_t specified_mem_size = 0, total_mem_size = 0; - struct memblock_region *reg; - bool first = true; phys_addr_t mem_start; phys_addr_t mem_end; + phys_addr_t reg_start, reg_end; unsigned int mem_max_regions; - int num, i; + bool first = true; + int num; + u64 i; /* Free-up PMSAv7_PROBE_REGION */ mpu_min_region_order = __mpu_min_region_order(); @@ -262,19 +263,19 @@ void __init pmsav7_adjust_lowmem_bounds(void) mem_max_regions -= num; #endif - for_each_memblock(memory, reg) { + for_each_mem_range(i, ®_start, ®_end) { if (first) { phys_addr_t phys_offset = PHYS_OFFSET; /* * Initially only use memory continuous from * PHYS_OFFSET */ - if (reg->base != phys_offset) + if (reg_start != phys_offset) panic("First memory bank must be contiguous from PHYS_OFFSET"); - mem_start = reg->base; - mem_end = reg->base + reg->size; - specified_mem_size = reg->size; + mem_start = reg_start; + mem_end = reg_end; + specified_mem_size = mem_end - mem_start; first = false; } else { /* @@ -283,8 +284,8 @@ void __init pmsav7_adjust_lowmem_bounds(void) * blocks separately while iterating) */ pr_notice("Ignoring RAM after %pa, memory at %pa ignored\n", - &mem_end, ®->base); - memblock_remove(reg->base, 0 - reg->base); + &mem_end, ®_start); + memblock_remove(reg_start, 0 - reg_start); break; } } diff --git a/arch/arm/mm/pmsa-v8.c b/arch/arm/mm/pmsa-v8.c index 0d7d5fb59247..28cdc5468406 100644 --- a/arch/arm/mm/pmsa-v8.c +++ b/arch/arm/mm/pmsa-v8.c @@ -11,7 +11,7 @@ #include <asm/cputype.h> #include <asm/mpu.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/sections.h> #include "mm.h" @@ -94,19 +94,20 @@ static __init bool is_region_fixed(int number) void __init pmsav8_adjust_lowmem_bounds(void) { phys_addr_t mem_end; - struct memblock_region *reg; + phys_addr_t reg_start, reg_end; bool first = true; + u64 i; - for_each_memblock(memory, reg) { + for_each_mem_range(i, ®_start, ®_end) { if (first) { phys_addr_t phys_offset = PHYS_OFFSET; /* * Initially only use memory continuous from * PHYS_OFFSET */ - if (reg->base != phys_offset) + if (reg_start != phys_offset) panic("First memory bank must be contiguous from PHYS_OFFSET"); - mem_end = reg->base + reg->size; + mem_end = reg_end; first = false; } else { /* @@ -115,8 +116,8 @@ void __init pmsav8_adjust_lowmem_bounds(void) * blocks separately while iterating) */ pr_notice("Ignoring RAM after %pa, memory at %pa ignored\n", - &mem_end, ®->base); - memblock_remove(reg->base, 0 - reg->base); + &mem_end, ®_start); + memblock_remove(reg_start, 0 - reg_start); break; } } diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S index 2785da387c91..d0ce3414a13e 100644 --- a/arch/arm/mm/proc-arm1020.S +++ b/arch/arm/mm/proc-arm1020.S @@ -11,11 +11,12 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -56,18 +57,20 @@ /* * cpu_arm1020_proc_init() */ -ENTRY(cpu_arm1020_proc_init) +SYM_TYPED_FUNC_START(cpu_arm1020_proc_init) ret lr +SYM_FUNC_END(cpu_arm1020_proc_init) /* * cpu_arm1020_proc_fin() */ -ENTRY(cpu_arm1020_proc_fin) +SYM_TYPED_FUNC_START(cpu_arm1020_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1000 @ ...i............ bic r0, r0, #0x000e @ ............wca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_arm1020_proc_fin) /* * cpu_arm1020_reset(loc) @@ -80,7 +83,7 @@ ENTRY(cpu_arm1020_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_arm1020_reset) +SYM_TYPED_FUNC_START(cpu_arm1020_reset) mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I,D caches mcr p15, 0, ip, c7, c10, 4 @ drain WB @@ -92,16 +95,17 @@ ENTRY(cpu_arm1020_reset) bic ip, ip, #0x1100 @ ...i...s........ mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_arm1020_reset) +SYM_FUNC_END(cpu_arm1020_reset) .popsection /* * cpu_arm1020_do_idle() */ .align 5 -ENTRY(cpu_arm1020_do_idle) +SYM_TYPED_FUNC_START(cpu_arm1020_do_idle) mcr p15, 0, r0, c7, c0, 4 @ Wait for interrupt ret lr +SYM_FUNC_END(cpu_arm1020_do_idle) /* ================================= CACHE ================================ */ @@ -112,13 +116,13 @@ ENTRY(cpu_arm1020_do_idle) * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(arm1020_flush_icache_all) +SYM_TYPED_FUNC_START(arm1020_flush_icache_all) #ifndef CONFIG_CPU_ICACHE_DISABLE mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache #endif ret lr -ENDPROC(arm1020_flush_icache_all) +SYM_FUNC_END(arm1020_flush_icache_all) /* * flush_user_cache_all() @@ -126,14 +130,14 @@ ENDPROC(arm1020_flush_icache_all) * Invalidate all cache entries in a particular address * space. */ -ENTRY(arm1020_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(arm1020_flush_user_cache_all, arm1020_flush_kern_cache_all) + /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(arm1020_flush_kern_cache_all) +SYM_TYPED_FUNC_START(arm1020_flush_kern_cache_all) mov r2, #VM_EXEC mov ip, #0 __flush_whole_cache: @@ -154,6 +158,7 @@ __flush_whole_cache: #endif mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1020_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -165,7 +170,7 @@ __flush_whole_cache: * - end - end address (exclusive) * - flags - vm_flags for this space */ -ENTRY(arm1020_flush_user_cache_range) +SYM_TYPED_FUNC_START(arm1020_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT @@ -185,6 +190,7 @@ ENTRY(arm1020_flush_user_cache_range) #endif mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1020_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -196,8 +202,11 @@ ENTRY(arm1020_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm1020_coherent_kern_range) - /* FALLTRHOUGH */ +SYM_TYPED_FUNC_START(arm1020_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b arm1020_coherent_user_range +#endif +SYM_FUNC_END(arm1020_coherent_kern_range) /* * coherent_user_range(start, end) @@ -209,7 +218,7 @@ ENTRY(arm1020_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm1020_coherent_user_range) +SYM_TYPED_FUNC_START(arm1020_coherent_user_range) mov ip, #0 bic r0, r0, #CACHE_DLINESIZE - 1 mcr p15, 0, ip, c7, c10, 4 @@ -227,6 +236,7 @@ ENTRY(arm1020_coherent_user_range) mcr p15, 0, ip, c7, c10, 4 @ drain WB mov r0, #0 ret lr +SYM_FUNC_END(arm1020_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -237,7 +247,7 @@ ENTRY(arm1020_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(arm1020_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(arm1020_flush_kern_dcache_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE add r1, r0, r1 @@ -249,6 +259,7 @@ ENTRY(arm1020_flush_kern_dcache_area) #endif mcr p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1020_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -314,7 +325,7 @@ arm1020_dma_clean_range: * - start - virtual start address * - end - virtual end address */ -ENTRY(arm1020_dma_flush_range) +SYM_TYPED_FUNC_START(arm1020_dma_flush_range) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE bic r0, r0, #CACHE_DLINESIZE - 1 @@ -327,6 +338,7 @@ ENTRY(arm1020_dma_flush_range) #endif mcr p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1020_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -334,13 +346,13 @@ ENTRY(arm1020_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(arm1020_dma_map_area) +SYM_TYPED_FUNC_START(arm1020_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq arm1020_dma_clean_range bcs arm1020_dma_inv_range b arm1020_dma_flush_range -ENDPROC(arm1020_dma_map_area) +SYM_FUNC_END(arm1020_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -348,18 +360,12 @@ ENDPROC(arm1020_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(arm1020_dma_unmap_area) +SYM_TYPED_FUNC_START(arm1020_dma_unmap_area) ret lr -ENDPROC(arm1020_dma_unmap_area) - - .globl arm1020_flush_kern_cache_louis - .equ arm1020_flush_kern_cache_louis, arm1020_flush_kern_cache_all - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions arm1020 +SYM_FUNC_END(arm1020_dma_unmap_area) .align 5 -ENTRY(cpu_arm1020_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_arm1020_dcache_clean_area) #ifndef CONFIG_CPU_DCACHE_DISABLE mov ip, #0 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry @@ -369,6 +375,7 @@ ENTRY(cpu_arm1020_dcache_clean_area) bhi 1b #endif ret lr +SYM_FUNC_END(cpu_arm1020_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -380,7 +387,7 @@ ENTRY(cpu_arm1020_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_arm1020_switch_mm) +SYM_TYPED_FUNC_START(cpu_arm1020_switch_mm) #ifdef CONFIG_MMU #ifndef CONFIG_CPU_DCACHE_DISABLE mcr p15, 0, r3, c7, c10, 4 @@ -408,14 +415,15 @@ ENTRY(cpu_arm1020_switch_mm) mcr p15, 0, r1, c8, c7, 0 @ invalidate I & D TLBs #endif /* CONFIG_MMU */ ret lr - +SYM_FUNC_END(cpu_arm1020_switch_mm) + /* * cpu_arm1020_set_pte(ptep, pte) * * Set a PTE and flush it out */ .align 5 -ENTRY(cpu_arm1020_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_arm1020_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext mov r0, r0 @@ -426,6 +434,7 @@ ENTRY(cpu_arm1020_set_pte_ext) mcr p15, 0, r0, c7, c10, 4 @ drain WB #endif /* CONFIG_MMU */ ret lr +SYM_FUNC_END(cpu_arm1020_set_pte_ext) .type __arm1020_setup, #function __arm1020_setup: diff --git a/arch/arm/mm/proc-arm1020e.S b/arch/arm/mm/proc-arm1020e.S index e9ea237ed785..64f031bf6eff 100644 --- a/arch/arm/mm/proc-arm1020e.S +++ b/arch/arm/mm/proc-arm1020e.S @@ -11,11 +11,12 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -56,18 +57,20 @@ /* * cpu_arm1020e_proc_init() */ -ENTRY(cpu_arm1020e_proc_init) +SYM_TYPED_FUNC_START(cpu_arm1020e_proc_init) ret lr +SYM_FUNC_END(cpu_arm1020e_proc_init) /* * cpu_arm1020e_proc_fin() */ -ENTRY(cpu_arm1020e_proc_fin) +SYM_TYPED_FUNC_START(cpu_arm1020e_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1000 @ ...i............ bic r0, r0, #0x000e @ ............wca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_arm1020e_proc_fin) /* * cpu_arm1020e_reset(loc) @@ -80,7 +83,7 @@ ENTRY(cpu_arm1020e_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_arm1020e_reset) +SYM_TYPED_FUNC_START(cpu_arm1020e_reset) mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I,D caches mcr p15, 0, ip, c7, c10, 4 @ drain WB @@ -92,16 +95,17 @@ ENTRY(cpu_arm1020e_reset) bic ip, ip, #0x1100 @ ...i...s........ mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_arm1020e_reset) +SYM_FUNC_END(cpu_arm1020e_reset) .popsection /* * cpu_arm1020e_do_idle() */ .align 5 -ENTRY(cpu_arm1020e_do_idle) +SYM_TYPED_FUNC_START(cpu_arm1020e_do_idle) mcr p15, 0, r0, c7, c0, 4 @ Wait for interrupt ret lr +SYM_FUNC_END(cpu_arm1020e_do_idle) /* ================================= CACHE ================================ */ @@ -112,13 +116,13 @@ ENTRY(cpu_arm1020e_do_idle) * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(arm1020e_flush_icache_all) +SYM_TYPED_FUNC_START(arm1020e_flush_icache_all) #ifndef CONFIG_CPU_ICACHE_DISABLE mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache #endif ret lr -ENDPROC(arm1020e_flush_icache_all) +SYM_FUNC_END(arm1020e_flush_icache_all) /* * flush_user_cache_all() @@ -126,14 +130,14 @@ ENDPROC(arm1020e_flush_icache_all) * Invalidate all cache entries in a particular address * space. */ -ENTRY(arm1020e_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(arm1020e_flush_user_cache_all, arm1020e_flush_kern_cache_all) + /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(arm1020e_flush_kern_cache_all) +SYM_TYPED_FUNC_START(arm1020e_flush_kern_cache_all) mov r2, #VM_EXEC mov ip, #0 __flush_whole_cache: @@ -153,6 +157,7 @@ __flush_whole_cache: #endif mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1020e_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -164,7 +169,7 @@ __flush_whole_cache: * - end - end address (exclusive) * - flags - vm_flags for this space */ -ENTRY(arm1020e_flush_user_cache_range) +SYM_TYPED_FUNC_START(arm1020e_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT @@ -182,6 +187,7 @@ ENTRY(arm1020e_flush_user_cache_range) #endif mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1020e_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -193,8 +199,12 @@ ENTRY(arm1020e_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm1020e_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(arm1020e_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b arm1020e_coherent_user_range +#endif +SYM_FUNC_END(arm1020e_coherent_kern_range) + /* * coherent_user_range(start, end) * @@ -205,7 +215,7 @@ ENTRY(arm1020e_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm1020e_coherent_user_range) +SYM_TYPED_FUNC_START(arm1020e_coherent_user_range) mov ip, #0 bic r0, r0, #CACHE_DLINESIZE - 1 1: @@ -221,6 +231,7 @@ ENTRY(arm1020e_coherent_user_range) mcr p15, 0, ip, c7, c10, 4 @ drain WB mov r0, #0 ret lr +SYM_FUNC_END(arm1020e_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -231,7 +242,7 @@ ENTRY(arm1020e_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(arm1020e_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(arm1020e_flush_kern_dcache_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE add r1, r0, r1 @@ -242,6 +253,7 @@ ENTRY(arm1020e_flush_kern_dcache_area) #endif mcr p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1020e_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -302,7 +314,7 @@ arm1020e_dma_clean_range: * - start - virtual start address * - end - virtual end address */ -ENTRY(arm1020e_dma_flush_range) +SYM_TYPED_FUNC_START(arm1020e_dma_flush_range) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE bic r0, r0, #CACHE_DLINESIZE - 1 @@ -313,6 +325,7 @@ ENTRY(arm1020e_dma_flush_range) #endif mcr p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1020e_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -320,13 +333,13 @@ ENTRY(arm1020e_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(arm1020e_dma_map_area) +SYM_TYPED_FUNC_START(arm1020e_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq arm1020e_dma_clean_range bcs arm1020e_dma_inv_range b arm1020e_dma_flush_range -ENDPROC(arm1020e_dma_map_area) +SYM_FUNC_END(arm1020e_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -334,18 +347,12 @@ ENDPROC(arm1020e_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(arm1020e_dma_unmap_area) +SYM_TYPED_FUNC_START(arm1020e_dma_unmap_area) ret lr -ENDPROC(arm1020e_dma_unmap_area) - - .globl arm1020e_flush_kern_cache_louis - .equ arm1020e_flush_kern_cache_louis, arm1020e_flush_kern_cache_all - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions arm1020e +SYM_FUNC_END(arm1020e_dma_unmap_area) .align 5 -ENTRY(cpu_arm1020e_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_arm1020e_dcache_clean_area) #ifndef CONFIG_CPU_DCACHE_DISABLE mov ip, #0 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry @@ -354,6 +361,7 @@ ENTRY(cpu_arm1020e_dcache_clean_area) bhi 1b #endif ret lr +SYM_FUNC_END(cpu_arm1020e_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -365,7 +373,7 @@ ENTRY(cpu_arm1020e_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_arm1020e_switch_mm) +SYM_TYPED_FUNC_START(cpu_arm1020e_switch_mm) #ifdef CONFIG_MMU #ifndef CONFIG_CPU_DCACHE_DISABLE mcr p15, 0, r3, c7, c10, 4 @@ -392,14 +400,15 @@ ENTRY(cpu_arm1020e_switch_mm) mcr p15, 0, r1, c8, c7, 0 @ invalidate I & D TLBs #endif ret lr - +SYM_FUNC_END(cpu_arm1020e_switch_mm) + /* * cpu_arm1020e_set_pte(ptep, pte) * * Set a PTE and flush it out */ .align 5 -ENTRY(cpu_arm1020e_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_arm1020e_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext mov r0, r0 @@ -408,6 +417,7 @@ ENTRY(cpu_arm1020e_set_pte_ext) #endif #endif /* CONFIG_MMU */ ret lr +SYM_FUNC_END(cpu_arm1020e_set_pte_ext) .type __arm1020e_setup, #function __arm1020e_setup: diff --git a/arch/arm/mm/proc-arm1022.S b/arch/arm/mm/proc-arm1022.S index 920c279e7879..42ed5ed07252 100644 --- a/arch/arm/mm/proc-arm1022.S +++ b/arch/arm/mm/proc-arm1022.S @@ -11,11 +11,12 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -56,18 +57,20 @@ /* * cpu_arm1022_proc_init() */ -ENTRY(cpu_arm1022_proc_init) +SYM_TYPED_FUNC_START(cpu_arm1022_proc_init) ret lr +SYM_FUNC_END(cpu_arm1022_proc_init) /* * cpu_arm1022_proc_fin() */ -ENTRY(cpu_arm1022_proc_fin) +SYM_TYPED_FUNC_START(cpu_arm1022_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1000 @ ...i............ bic r0, r0, #0x000e @ ............wca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_arm1022_proc_fin) /* * cpu_arm1022_reset(loc) @@ -80,7 +83,7 @@ ENTRY(cpu_arm1022_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_arm1022_reset) +SYM_TYPED_FUNC_START(cpu_arm1022_reset) mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I,D caches mcr p15, 0, ip, c7, c10, 4 @ drain WB @@ -92,16 +95,17 @@ ENTRY(cpu_arm1022_reset) bic ip, ip, #0x1100 @ ...i...s........ mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_arm1022_reset) +SYM_FUNC_END(cpu_arm1022_reset) .popsection /* * cpu_arm1022_do_idle() */ .align 5 -ENTRY(cpu_arm1022_do_idle) +SYM_TYPED_FUNC_START(cpu_arm1022_do_idle) mcr p15, 0, r0, c7, c0, 4 @ Wait for interrupt ret lr +SYM_FUNC_END(cpu_arm1022_do_idle) /* ================================= CACHE ================================ */ @@ -112,13 +116,13 @@ ENTRY(cpu_arm1022_do_idle) * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(arm1022_flush_icache_all) +SYM_TYPED_FUNC_START(arm1022_flush_icache_all) #ifndef CONFIG_CPU_ICACHE_DISABLE mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache #endif ret lr -ENDPROC(arm1022_flush_icache_all) +SYM_FUNC_END(arm1022_flush_icache_all) /* * flush_user_cache_all() @@ -126,14 +130,14 @@ ENDPROC(arm1022_flush_icache_all) * Invalidate all cache entries in a particular address * space. */ -ENTRY(arm1022_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(arm1022_flush_user_cache_all, arm1022_flush_kern_cache_all) + /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(arm1022_flush_kern_cache_all) +SYM_TYPED_FUNC_START(arm1022_flush_kern_cache_all) mov r2, #VM_EXEC mov ip, #0 __flush_whole_cache: @@ -152,6 +156,7 @@ __flush_whole_cache: #endif mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1022_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -163,7 +168,7 @@ __flush_whole_cache: * - end - end address (exclusive) * - flags - vm_flags for this space */ -ENTRY(arm1022_flush_user_cache_range) +SYM_TYPED_FUNC_START(arm1022_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT @@ -181,6 +186,7 @@ ENTRY(arm1022_flush_user_cache_range) #endif mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1022_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -192,8 +198,11 @@ ENTRY(arm1022_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm1022_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(arm1022_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b arm1022_coherent_user_range +#endif +SYM_FUNC_END(arm1022_coherent_kern_range) /* * coherent_user_range(start, end) @@ -205,7 +214,7 @@ ENTRY(arm1022_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm1022_coherent_user_range) +SYM_TYPED_FUNC_START(arm1022_coherent_user_range) mov ip, #0 bic r0, r0, #CACHE_DLINESIZE - 1 1: @@ -221,6 +230,7 @@ ENTRY(arm1022_coherent_user_range) mcr p15, 0, ip, c7, c10, 4 @ drain WB mov r0, #0 ret lr +SYM_FUNC_END(arm1022_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -231,7 +241,7 @@ ENTRY(arm1022_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(arm1022_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(arm1022_flush_kern_dcache_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE add r1, r0, r1 @@ -242,6 +252,7 @@ ENTRY(arm1022_flush_kern_dcache_area) #endif mcr p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1022_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -302,7 +313,7 @@ arm1022_dma_clean_range: * - start - virtual start address * - end - virtual end address */ -ENTRY(arm1022_dma_flush_range) +SYM_TYPED_FUNC_START(arm1022_dma_flush_range) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE bic r0, r0, #CACHE_DLINESIZE - 1 @@ -313,6 +324,7 @@ ENTRY(arm1022_dma_flush_range) #endif mcr p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1022_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -320,13 +332,13 @@ ENTRY(arm1022_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(arm1022_dma_map_area) +SYM_TYPED_FUNC_START(arm1022_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq arm1022_dma_clean_range bcs arm1022_dma_inv_range b arm1022_dma_flush_range -ENDPROC(arm1022_dma_map_area) +SYM_FUNC_END(arm1022_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -334,18 +346,12 @@ ENDPROC(arm1022_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(arm1022_dma_unmap_area) +SYM_TYPED_FUNC_START(arm1022_dma_unmap_area) ret lr -ENDPROC(arm1022_dma_unmap_area) - - .globl arm1022_flush_kern_cache_louis - .equ arm1022_flush_kern_cache_louis, arm1022_flush_kern_cache_all - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions arm1022 +SYM_FUNC_END(arm1022_dma_unmap_area) .align 5 -ENTRY(cpu_arm1022_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_arm1022_dcache_clean_area) #ifndef CONFIG_CPU_DCACHE_DISABLE mov ip, #0 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry @@ -354,6 +360,7 @@ ENTRY(cpu_arm1022_dcache_clean_area) bhi 1b #endif ret lr +SYM_FUNC_END(cpu_arm1022_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -365,7 +372,7 @@ ENTRY(cpu_arm1022_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_arm1022_switch_mm) +SYM_TYPED_FUNC_START(cpu_arm1022_switch_mm) #ifdef CONFIG_MMU #ifndef CONFIG_CPU_DCACHE_DISABLE mov r1, #(CACHE_DSEGMENTS - 1) << 5 @ 16 segments @@ -385,14 +392,15 @@ ENTRY(cpu_arm1022_switch_mm) mcr p15, 0, r1, c8, c7, 0 @ invalidate I & D TLBs #endif ret lr - +SYM_FUNC_END(cpu_arm1022_switch_mm) + /* * cpu_arm1022_set_pte_ext(ptep, pte, ext) * * Set a PTE and flush it out */ .align 5 -ENTRY(cpu_arm1022_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_arm1022_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext mov r0, r0 @@ -401,6 +409,7 @@ ENTRY(cpu_arm1022_set_pte_ext) #endif #endif /* CONFIG_MMU */ ret lr +SYM_FUNC_END(cpu_arm1022_set_pte_ext) .type __arm1022_setup, #function __arm1022_setup: diff --git a/arch/arm/mm/proc-arm1026.S b/arch/arm/mm/proc-arm1026.S index 0bdf25a95b10..b3ae62cd553a 100644 --- a/arch/arm/mm/proc-arm1026.S +++ b/arch/arm/mm/proc-arm1026.S @@ -11,11 +11,12 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -56,18 +57,20 @@ /* * cpu_arm1026_proc_init() */ -ENTRY(cpu_arm1026_proc_init) +SYM_TYPED_FUNC_START(cpu_arm1026_proc_init) ret lr +SYM_FUNC_END(cpu_arm1026_proc_init) /* * cpu_arm1026_proc_fin() */ -ENTRY(cpu_arm1026_proc_fin) +SYM_TYPED_FUNC_START(cpu_arm1026_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1000 @ ...i............ bic r0, r0, #0x000e @ ............wca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_arm1026_proc_fin) /* * cpu_arm1026_reset(loc) @@ -80,7 +83,7 @@ ENTRY(cpu_arm1026_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_arm1026_reset) +SYM_TYPED_FUNC_START(cpu_arm1026_reset) mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I,D caches mcr p15, 0, ip, c7, c10, 4 @ drain WB @@ -92,16 +95,17 @@ ENTRY(cpu_arm1026_reset) bic ip, ip, #0x1100 @ ...i...s........ mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_arm1026_reset) +SYM_FUNC_END(cpu_arm1026_reset) .popsection /* * cpu_arm1026_do_idle() */ .align 5 -ENTRY(cpu_arm1026_do_idle) +SYM_TYPED_FUNC_START(cpu_arm1026_do_idle) mcr p15, 0, r0, c7, c0, 4 @ Wait for interrupt ret lr +SYM_FUNC_END(cpu_arm1026_do_idle) /* ================================= CACHE ================================ */ @@ -112,13 +116,13 @@ ENTRY(cpu_arm1026_do_idle) * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(arm1026_flush_icache_all) +SYM_TYPED_FUNC_START(arm1026_flush_icache_all) #ifndef CONFIG_CPU_ICACHE_DISABLE mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache #endif ret lr -ENDPROC(arm1026_flush_icache_all) +SYM_FUNC_END(arm1026_flush_icache_all) /* * flush_user_cache_all() @@ -126,14 +130,14 @@ ENDPROC(arm1026_flush_icache_all) * Invalidate all cache entries in a particular address * space. */ -ENTRY(arm1026_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(arm1026_flush_user_cache_all, arm1026_flush_kern_cache_all) + /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(arm1026_flush_kern_cache_all) +SYM_TYPED_FUNC_START(arm1026_flush_kern_cache_all) mov r2, #VM_EXEC mov ip, #0 __flush_whole_cache: @@ -147,6 +151,7 @@ __flush_whole_cache: #endif mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1026_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -158,7 +163,7 @@ __flush_whole_cache: * - end - end address (exclusive) * - flags - vm_flags for this space */ -ENTRY(arm1026_flush_user_cache_range) +SYM_TYPED_FUNC_START(arm1026_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT @@ -176,6 +181,7 @@ ENTRY(arm1026_flush_user_cache_range) #endif mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1026_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -187,8 +193,12 @@ ENTRY(arm1026_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm1026_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(arm1026_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b arm1026_coherent_user_range +#endif +SYM_FUNC_END(arm1026_coherent_kern_range) + /* * coherent_user_range(start, end) * @@ -199,7 +209,7 @@ ENTRY(arm1026_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm1026_coherent_user_range) +SYM_TYPED_FUNC_START(arm1026_coherent_user_range) mov ip, #0 bic r0, r0, #CACHE_DLINESIZE - 1 1: @@ -215,6 +225,7 @@ ENTRY(arm1026_coherent_user_range) mcr p15, 0, ip, c7, c10, 4 @ drain WB mov r0, #0 ret lr +SYM_FUNC_END(arm1026_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -225,7 +236,7 @@ ENTRY(arm1026_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(arm1026_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(arm1026_flush_kern_dcache_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE add r1, r0, r1 @@ -236,6 +247,7 @@ ENTRY(arm1026_flush_kern_dcache_area) #endif mcr p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1026_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -296,7 +308,7 @@ arm1026_dma_clean_range: * - start - virtual start address * - end - virtual end address */ -ENTRY(arm1026_dma_flush_range) +SYM_TYPED_FUNC_START(arm1026_dma_flush_range) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE bic r0, r0, #CACHE_DLINESIZE - 1 @@ -307,6 +319,7 @@ ENTRY(arm1026_dma_flush_range) #endif mcr p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm1026_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -314,13 +327,13 @@ ENTRY(arm1026_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(arm1026_dma_map_area) +SYM_TYPED_FUNC_START(arm1026_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq arm1026_dma_clean_range bcs arm1026_dma_inv_range b arm1026_dma_flush_range -ENDPROC(arm1026_dma_map_area) +SYM_FUNC_END(arm1026_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -328,18 +341,12 @@ ENDPROC(arm1026_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(arm1026_dma_unmap_area) +SYM_TYPED_FUNC_START(arm1026_dma_unmap_area) ret lr -ENDPROC(arm1026_dma_unmap_area) - - .globl arm1026_flush_kern_cache_louis - .equ arm1026_flush_kern_cache_louis, arm1026_flush_kern_cache_all - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions arm1026 +SYM_FUNC_END(arm1026_dma_unmap_area) .align 5 -ENTRY(cpu_arm1026_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_arm1026_dcache_clean_area) #ifndef CONFIG_CPU_DCACHE_DISABLE mov ip, #0 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry @@ -348,6 +355,7 @@ ENTRY(cpu_arm1026_dcache_clean_area) bhi 1b #endif ret lr +SYM_FUNC_END(cpu_arm1026_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -359,7 +367,7 @@ ENTRY(cpu_arm1026_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_arm1026_switch_mm) +SYM_TYPED_FUNC_START(cpu_arm1026_switch_mm) #ifdef CONFIG_MMU mov r1, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE @@ -374,14 +382,15 @@ ENTRY(cpu_arm1026_switch_mm) mcr p15, 0, r1, c8, c7, 0 @ invalidate I & D TLBs #endif ret lr - +SYM_FUNC_END(cpu_arm1026_switch_mm) + /* * cpu_arm1026_set_pte_ext(ptep, pte, ext) * * Set a PTE and flush it out */ .align 5 -ENTRY(cpu_arm1026_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_arm1026_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext mov r0, r0 @@ -390,6 +399,7 @@ ENTRY(cpu_arm1026_set_pte_ext) #endif #endif /* CONFIG_MMU */ ret lr +SYM_FUNC_END(cpu_arm1026_set_pte_ext) .type __arm1026_setup, #function __arm1026_setup: diff --git a/arch/arm/mm/proc-arm720.S b/arch/arm/mm/proc-arm720.S index 39361e196d61..59732c334e1d 100644 --- a/arch/arm/mm/proc-arm720.S +++ b/arch/arm/mm/proc-arm720.S @@ -20,11 +20,12 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -35,24 +36,30 @@ * * Notes : This processor does not require these */ -ENTRY(cpu_arm720_dcache_clean_area) -ENTRY(cpu_arm720_proc_init) +SYM_TYPED_FUNC_START(cpu_arm720_dcache_clean_area) ret lr +SYM_FUNC_END(cpu_arm720_dcache_clean_area) -ENTRY(cpu_arm720_proc_fin) +SYM_TYPED_FUNC_START(cpu_arm720_proc_init) + ret lr +SYM_FUNC_END(cpu_arm720_proc_init) + +SYM_TYPED_FUNC_START(cpu_arm720_proc_fin) mrc p15, 0, r0, c1, c0, 0 bic r0, r0, #0x1000 @ ...i............ bic r0, r0, #0x000e @ ............wca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_arm720_proc_fin) /* * Function: arm720_proc_do_idle(void) * Params : r0 = unused * Purpose : put the processor in proper idle mode */ -ENTRY(cpu_arm720_do_idle) +SYM_TYPED_FUNC_START(cpu_arm720_do_idle) ret lr +SYM_FUNC_END(cpu_arm720_do_idle) /* * Function: arm720_switch_mm(unsigned long pgd_phys) @@ -60,7 +67,7 @@ ENTRY(cpu_arm720_do_idle) * Purpose : Perform a task switch, saving the old process' state and restoring * the new. */ -ENTRY(cpu_arm720_switch_mm) +SYM_TYPED_FUNC_START(cpu_arm720_switch_mm) #ifdef CONFIG_MMU mov r1, #0 mcr p15, 0, r1, c7, c7, 0 @ invalidate cache @@ -68,6 +75,7 @@ ENTRY(cpu_arm720_switch_mm) mcr p15, 0, r1, c8, c7, 0 @ flush TLB (v4) #endif ret lr +SYM_FUNC_END(cpu_arm720_switch_mm) /* * Function: arm720_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext) @@ -76,11 +84,12 @@ ENTRY(cpu_arm720_switch_mm) * Purpose : Set a PTE and flush it out of any WB cache */ .align 5 -ENTRY(cpu_arm720_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_arm720_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext wc_disable=0 #endif ret lr +SYM_FUNC_END(cpu_arm720_set_pte_ext) /* * Function: arm720_reset @@ -88,7 +97,7 @@ ENTRY(cpu_arm720_set_pte_ext) * Notes : This sets up everything for a reset */ .pushsection .idmap.text, "ax" -ENTRY(cpu_arm720_reset) +SYM_TYPED_FUNC_START(cpu_arm720_reset) mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate cache #ifdef CONFIG_MMU @@ -99,7 +108,7 @@ ENTRY(cpu_arm720_reset) bic ip, ip, #0x2100 @ ..v....s........ mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_arm720_reset) +SYM_FUNC_END(cpu_arm720_reset) .popsection .type __arm710_setup, #function diff --git a/arch/arm/mm/proc-arm740.S b/arch/arm/mm/proc-arm740.S index 1a94bbf6e53f..78854df63964 100644 --- a/arch/arm/mm/proc-arm740.S +++ b/arch/arm/mm/proc-arm740.S @@ -6,11 +6,12 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -24,21 +25,32 @@ * * These are not required. */ -ENTRY(cpu_arm740_proc_init) -ENTRY(cpu_arm740_do_idle) -ENTRY(cpu_arm740_dcache_clean_area) -ENTRY(cpu_arm740_switch_mm) +SYM_TYPED_FUNC_START(cpu_arm740_proc_init) ret lr +SYM_FUNC_END(cpu_arm740_proc_init) + +SYM_TYPED_FUNC_START(cpu_arm740_do_idle) + ret lr +SYM_FUNC_END(cpu_arm740_do_idle) + +SYM_TYPED_FUNC_START(cpu_arm740_dcache_clean_area) + ret lr +SYM_FUNC_END(cpu_arm740_dcache_clean_area) + +SYM_TYPED_FUNC_START(cpu_arm740_switch_mm) + ret lr +SYM_FUNC_END(cpu_arm740_switch_mm) /* * cpu_arm740_proc_fin() */ -ENTRY(cpu_arm740_proc_fin) +SYM_TYPED_FUNC_START(cpu_arm740_proc_fin) mrc p15, 0, r0, c1, c0, 0 bic r0, r0, #0x3f000000 @ bank/f/lock/s bic r0, r0, #0x0000000c @ w-buffer/cache mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_arm740_proc_fin) /* * cpu_arm740_reset(loc) @@ -46,14 +58,14 @@ ENTRY(cpu_arm740_proc_fin) * Notes : This sets up everything for a reset */ .pushsection .idmap.text, "ax" -ENTRY(cpu_arm740_reset) +SYM_TYPED_FUNC_START(cpu_arm740_reset) mov ip, #0 mcr p15, 0, ip, c7, c0, 0 @ invalidate cache mrc p15, 0, ip, c1, c0, 0 @ get ctrl register bic ip, ip, #0x0000000c @ ............wc.. mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_arm740_reset) +SYM_FUNC_END(cpu_arm740_reset) .popsection .type __arm740_setup, #function diff --git a/arch/arm/mm/proc-arm7tdmi.S b/arch/arm/mm/proc-arm7tdmi.S index 52b66cf0259e..baa3d4472147 100644 --- a/arch/arm/mm/proc-arm7tdmi.S +++ b/arch/arm/mm/proc-arm7tdmi.S @@ -6,11 +6,12 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -23,18 +24,29 @@ * cpu_arm7tdmi_switch_mm() * * These are not required. - */ -ENTRY(cpu_arm7tdmi_proc_init) -ENTRY(cpu_arm7tdmi_do_idle) -ENTRY(cpu_arm7tdmi_dcache_clean_area) -ENTRY(cpu_arm7tdmi_switch_mm) - ret lr +*/ +SYM_TYPED_FUNC_START(cpu_arm7tdmi_proc_init) + ret lr +SYM_FUNC_END(cpu_arm7tdmi_proc_init) + +SYM_TYPED_FUNC_START(cpu_arm7tdmi_do_idle) + ret lr +SYM_FUNC_END(cpu_arm7tdmi_do_idle) + +SYM_TYPED_FUNC_START(cpu_arm7tdmi_dcache_clean_area) + ret lr +SYM_FUNC_END(cpu_arm7tdmi_dcache_clean_area) + +SYM_TYPED_FUNC_START(cpu_arm7tdmi_switch_mm) + ret lr +SYM_FUNC_END(cpu_arm7tdmi_switch_mm) /* * cpu_arm7tdmi_proc_fin() - */ -ENTRY(cpu_arm7tdmi_proc_fin) - ret lr +*/ +SYM_TYPED_FUNC_START(cpu_arm7tdmi_proc_fin) + ret lr +SYM_FUNC_END(cpu_arm7tdmi_proc_fin) /* * Function: cpu_arm7tdmi_reset(loc) @@ -42,9 +54,9 @@ ENTRY(cpu_arm7tdmi_proc_fin) * Purpose : Sets up everything for a reset and jump to the location for soft reset. */ .pushsection .idmap.text, "ax" -ENTRY(cpu_arm7tdmi_reset) +SYM_TYPED_FUNC_START(cpu_arm7tdmi_reset) ret r0 -ENDPROC(cpu_arm7tdmi_reset) +SYM_FUNC_END(cpu_arm7tdmi_reset) .popsection .type __arm7tdmi_setup, #function diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S index 31ac8acc34dc..a30df54ad5fa 100644 --- a/arch/arm/mm/proc-arm920.S +++ b/arch/arm/mm/proc-arm920.S @@ -13,10 +13,11 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/page.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -48,18 +49,20 @@ /* * cpu_arm920_proc_init() */ -ENTRY(cpu_arm920_proc_init) +SYM_TYPED_FUNC_START(cpu_arm920_proc_init) ret lr +SYM_FUNC_END(cpu_arm920_proc_init) /* * cpu_arm920_proc_fin() */ -ENTRY(cpu_arm920_proc_fin) +SYM_TYPED_FUNC_START(cpu_arm920_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1000 @ ...i............ bic r0, r0, #0x000e @ ............wca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_arm920_proc_fin) /* * cpu_arm920_reset(loc) @@ -72,7 +75,7 @@ ENTRY(cpu_arm920_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_arm920_reset) +SYM_TYPED_FUNC_START(cpu_arm920_reset) mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I,D caches mcr p15, 0, ip, c7, c10, 4 @ drain WB @@ -84,17 +87,17 @@ ENTRY(cpu_arm920_reset) bic ip, ip, #0x1100 @ ...i...s........ mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_arm920_reset) +SYM_FUNC_END(cpu_arm920_reset) .popsection /* * cpu_arm920_do_idle() */ .align 5 -ENTRY(cpu_arm920_do_idle) +SYM_TYPED_FUNC_START(cpu_arm920_do_idle) mcr p15, 0, r0, c7, c0, 4 @ Wait for interrupt ret lr - +SYM_FUNC_END(cpu_arm920_do_idle) #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH @@ -103,11 +106,11 @@ ENTRY(cpu_arm920_do_idle) * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(arm920_flush_icache_all) +SYM_TYPED_FUNC_START(arm920_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(arm920_flush_icache_all) +SYM_FUNC_END(arm920_flush_icache_all) /* * flush_user_cache_all() @@ -115,15 +118,14 @@ ENDPROC(arm920_flush_icache_all) * Invalidate all cache entries in a particular address * space. */ -ENTRY(arm920_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(arm920_flush_user_cache_all, arm920_flush_kern_cache_all) /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(arm920_flush_kern_cache_all) +SYM_TYPED_FUNC_START(arm920_flush_kern_cache_all) mov r2, #VM_EXEC mov ip, #0 __flush_whole_cache: @@ -138,6 +140,7 @@ __flush_whole_cache: mcrne p15, 0, ip, c7, c5, 0 @ invalidate I cache mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm920_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -149,7 +152,7 @@ __flush_whole_cache: * - end - end address (exclusive) * - flags - vm_flags for address space */ -ENTRY(arm920_flush_user_cache_range) +SYM_TYPED_FUNC_START(arm920_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT @@ -164,6 +167,7 @@ ENTRY(arm920_flush_user_cache_range) tst r2, #VM_EXEC mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm920_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -175,8 +179,11 @@ ENTRY(arm920_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm920_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(arm920_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b arm920_coherent_user_range +#endif +SYM_FUNC_END(arm920_coherent_kern_range) /* * coherent_user_range(start, end) @@ -188,7 +195,7 @@ ENTRY(arm920_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm920_coherent_user_range) +SYM_TYPED_FUNC_START(arm920_coherent_user_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry mcr p15, 0, r0, c7, c5, 1 @ invalidate I entry @@ -198,6 +205,7 @@ ENTRY(arm920_coherent_user_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov r0, #0 ret lr +SYM_FUNC_END(arm920_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -208,7 +216,7 @@ ENTRY(arm920_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(arm920_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(arm920_flush_kern_dcache_area) add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE @@ -218,6 +226,7 @@ ENTRY(arm920_flush_kern_dcache_area) mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm920_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -272,7 +281,7 @@ arm920_dma_clean_range: * - start - virtual start address * - end - virtual end address */ -ENTRY(arm920_dma_flush_range) +SYM_TYPED_FUNC_START(arm920_dma_flush_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE @@ -280,6 +289,7 @@ ENTRY(arm920_dma_flush_range) blo 1b mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm920_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -287,13 +297,13 @@ ENTRY(arm920_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(arm920_dma_map_area) +SYM_TYPED_FUNC_START(arm920_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq arm920_dma_clean_range bcs arm920_dma_inv_range b arm920_dma_flush_range -ENDPROC(arm920_dma_map_area) +SYM_FUNC_END(arm920_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -301,24 +311,20 @@ ENDPROC(arm920_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(arm920_dma_unmap_area) +SYM_TYPED_FUNC_START(arm920_dma_unmap_area) ret lr -ENDPROC(arm920_dma_unmap_area) +SYM_FUNC_END(arm920_dma_unmap_area) - .globl arm920_flush_kern_cache_louis - .equ arm920_flush_kern_cache_louis, arm920_flush_kern_cache_all - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions arm920 -#endif +#endif /* !CONFIG_CPU_DCACHE_WRITETHROUGH */ -ENTRY(cpu_arm920_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_arm920_dcache_clean_area) 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHE_DLINESIZE subs r1, r1, #CACHE_DLINESIZE bhi 1b ret lr +SYM_FUNC_END(cpu_arm920_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -330,7 +336,7 @@ ENTRY(cpu_arm920_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_arm920_switch_mm) +SYM_TYPED_FUNC_START(cpu_arm920_switch_mm) #ifdef CONFIG_MMU mov ip, #0 #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH @@ -354,6 +360,7 @@ ENTRY(cpu_arm920_switch_mm) mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs #endif ret lr +SYM_FUNC_END(cpu_arm920_switch_mm) /* * cpu_arm920_set_pte(ptep, pte, ext) @@ -361,7 +368,7 @@ ENTRY(cpu_arm920_switch_mm) * Set a PTE and flush it out */ .align 5 -ENTRY(cpu_arm920_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_arm920_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext mov r0, r0 @@ -369,21 +376,22 @@ ENTRY(cpu_arm920_set_pte_ext) mcr p15, 0, r0, c7, c10, 4 @ drain WB #endif ret lr +SYM_FUNC_END(cpu_arm920_set_pte_ext) /* Suspend/resume support: taken from arch/arm/plat-s3c24xx/sleep.S */ .globl cpu_arm920_suspend_size .equ cpu_arm920_suspend_size, 4 * 3 #ifdef CONFIG_ARM_CPU_SUSPEND -ENTRY(cpu_arm920_do_suspend) +SYM_TYPED_FUNC_START(cpu_arm920_do_suspend) stmfd sp!, {r4 - r6, lr} mrc p15, 0, r4, c13, c0, 0 @ PID mrc p15, 0, r5, c3, c0, 0 @ Domain ID mrc p15, 0, r6, c1, c0, 0 @ Control register stmia r0, {r4 - r6} ldmfd sp!, {r4 - r6, pc} -ENDPROC(cpu_arm920_do_suspend) +SYM_FUNC_END(cpu_arm920_do_suspend) -ENTRY(cpu_arm920_do_resume) +SYM_TYPED_FUNC_START(cpu_arm920_do_resume) mov ip, #0 mcr p15, 0, ip, c8, c7, 0 @ invalidate I+D TLBs mcr p15, 0, ip, c7, c7, 0 @ invalidate I+D caches @@ -393,7 +401,7 @@ ENTRY(cpu_arm920_do_resume) mcr p15, 0, r1, c2, c0, 0 @ TTB address mov r0, r6 @ control register b cpu_resume_mmu -ENDPROC(cpu_arm920_do_resume) +SYM_FUNC_END(cpu_arm920_do_resume) #endif .type __arm920_setup, #function diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S index ca2c7ca8af21..aac4e048100d 100644 --- a/arch/arm/mm/proc-arm922.S +++ b/arch/arm/mm/proc-arm922.S @@ -14,10 +14,11 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/page.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -50,18 +51,20 @@ /* * cpu_arm922_proc_init() */ -ENTRY(cpu_arm922_proc_init) +SYM_TYPED_FUNC_START(cpu_arm922_proc_init) ret lr +SYM_FUNC_END(cpu_arm922_proc_init) /* * cpu_arm922_proc_fin() */ -ENTRY(cpu_arm922_proc_fin) +SYM_TYPED_FUNC_START(cpu_arm922_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1000 @ ...i............ bic r0, r0, #0x000e @ ............wca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_arm922_proc_fin) /* * cpu_arm922_reset(loc) @@ -74,7 +77,7 @@ ENTRY(cpu_arm922_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_arm922_reset) +SYM_TYPED_FUNC_START(cpu_arm922_reset) mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I,D caches mcr p15, 0, ip, c7, c10, 4 @ drain WB @@ -86,17 +89,17 @@ ENTRY(cpu_arm922_reset) bic ip, ip, #0x1100 @ ...i...s........ mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_arm922_reset) +SYM_FUNC_END(cpu_arm922_reset) .popsection /* * cpu_arm922_do_idle() */ .align 5 -ENTRY(cpu_arm922_do_idle) +SYM_TYPED_FUNC_START(cpu_arm922_do_idle) mcr p15, 0, r0, c7, c0, 4 @ Wait for interrupt ret lr - +SYM_FUNC_END(cpu_arm922_do_idle) #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH @@ -105,11 +108,11 @@ ENTRY(cpu_arm922_do_idle) * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(arm922_flush_icache_all) +SYM_TYPED_FUNC_START(arm922_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(arm922_flush_icache_all) +SYM_FUNC_END(arm922_flush_icache_all) /* * flush_user_cache_all() @@ -117,15 +120,14 @@ ENDPROC(arm922_flush_icache_all) * Clean and invalidate all cache entries in a particular * address space. */ -ENTRY(arm922_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(arm922_flush_user_cache_all, arm922_flush_kern_cache_all) /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(arm922_flush_kern_cache_all) +SYM_TYPED_FUNC_START(arm922_flush_kern_cache_all) mov r2, #VM_EXEC mov ip, #0 __flush_whole_cache: @@ -140,6 +142,7 @@ __flush_whole_cache: mcrne p15, 0, ip, c7, c5, 0 @ invalidate I cache mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm922_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -151,7 +154,7 @@ __flush_whole_cache: * - end - end address (exclusive) * - flags - vm_flags describing address space */ -ENTRY(arm922_flush_user_cache_range) +SYM_TYPED_FUNC_START(arm922_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT @@ -166,6 +169,7 @@ ENTRY(arm922_flush_user_cache_range) tst r2, #VM_EXEC mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm922_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -177,8 +181,11 @@ ENTRY(arm922_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm922_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(arm922_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b arm922_coherent_user_range +#endif +SYM_FUNC_END(arm922_coherent_kern_range) /* * coherent_user_range(start, end) @@ -190,7 +197,7 @@ ENTRY(arm922_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm922_coherent_user_range) +SYM_TYPED_FUNC_START(arm922_coherent_user_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry mcr p15, 0, r0, c7, c5, 1 @ invalidate I entry @@ -200,6 +207,7 @@ ENTRY(arm922_coherent_user_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov r0, #0 ret lr +SYM_FUNC_END(arm922_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -210,7 +218,7 @@ ENTRY(arm922_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(arm922_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(arm922_flush_kern_dcache_area) add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE @@ -220,6 +228,7 @@ ENTRY(arm922_flush_kern_dcache_area) mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm922_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -274,7 +283,7 @@ arm922_dma_clean_range: * - start - virtual start address * - end - virtual end address */ -ENTRY(arm922_dma_flush_range) +SYM_TYPED_FUNC_START(arm922_dma_flush_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE @@ -282,6 +291,7 @@ ENTRY(arm922_dma_flush_range) blo 1b mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm922_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -289,13 +299,13 @@ ENTRY(arm922_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(arm922_dma_map_area) +SYM_TYPED_FUNC_START(arm922_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq arm922_dma_clean_range bcs arm922_dma_inv_range b arm922_dma_flush_range -ENDPROC(arm922_dma_map_area) +SYM_FUNC_END(arm922_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -303,19 +313,13 @@ ENDPROC(arm922_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(arm922_dma_unmap_area) +SYM_TYPED_FUNC_START(arm922_dma_unmap_area) ret lr -ENDPROC(arm922_dma_unmap_area) - - .globl arm922_flush_kern_cache_louis - .equ arm922_flush_kern_cache_louis, arm922_flush_kern_cache_all - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions arm922 -#endif +SYM_FUNC_END(arm922_dma_unmap_area) +#endif /* !CONFIG_CPU_DCACHE_WRITETHROUGH */ -ENTRY(cpu_arm922_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_arm922_dcache_clean_area) #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHE_DLINESIZE @@ -323,6 +327,7 @@ ENTRY(cpu_arm922_dcache_clean_area) bhi 1b #endif ret lr +SYM_FUNC_END(cpu_arm922_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -334,7 +339,7 @@ ENTRY(cpu_arm922_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_arm922_switch_mm) +SYM_TYPED_FUNC_START(cpu_arm922_switch_mm) #ifdef CONFIG_MMU mov ip, #0 #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH @@ -358,6 +363,7 @@ ENTRY(cpu_arm922_switch_mm) mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs #endif ret lr +SYM_FUNC_END(cpu_arm922_switch_mm) /* * cpu_arm922_set_pte_ext(ptep, pte, ext) @@ -365,7 +371,7 @@ ENTRY(cpu_arm922_switch_mm) * Set a PTE and flush it out */ .align 5 -ENTRY(cpu_arm922_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_arm922_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext mov r0, r0 @@ -373,6 +379,7 @@ ENTRY(cpu_arm922_set_pte_ext) mcr p15, 0, r0, c7, c10, 4 @ drain WB #endif /* CONFIG_MMU */ ret lr +SYM_FUNC_END(cpu_arm922_set_pte_ext) .type __arm922_setup, #function __arm922_setup: diff --git a/arch/arm/mm/proc-arm925.S b/arch/arm/mm/proc-arm925.S index a381a0c9f109..035941faeb2e 100644 --- a/arch/arm/mm/proc-arm925.S +++ b/arch/arm/mm/proc-arm925.S @@ -37,10 +37,11 @@ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/page.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -71,18 +72,20 @@ /* * cpu_arm925_proc_init() */ -ENTRY(cpu_arm925_proc_init) +SYM_TYPED_FUNC_START(cpu_arm925_proc_init) ret lr +SYM_FUNC_END(cpu_arm925_proc_init) /* * cpu_arm925_proc_fin() */ -ENTRY(cpu_arm925_proc_fin) +SYM_TYPED_FUNC_START(cpu_arm925_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1000 @ ...i............ bic r0, r0, #0x000e @ ............wca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_arm925_proc_fin) /* * cpu_arm925_reset(loc) @@ -95,14 +98,14 @@ ENTRY(cpu_arm925_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_arm925_reset) +SYM_TYPED_FUNC_START(cpu_arm925_reset) /* Send software reset to MPU and DSP */ mov ip, #0xff000000 orr ip, ip, #0x00fe0000 orr ip, ip, #0x0000ce00 mov r4, #1 strh r4, [ip, #0x10] -ENDPROC(cpu_arm925_reset) +SYM_FUNC_END(cpu_arm925_reset) .popsection mov ip, #0 @@ -123,7 +126,7 @@ ENDPROC(cpu_arm925_reset) * Called with IRQs disabled */ .align 10 -ENTRY(cpu_arm925_do_idle) +SYM_TYPED_FUNC_START(cpu_arm925_do_idle) mov r0, #0 mrc p15, 0, r1, c1, c0, 0 @ Read control register mcr p15, 0, r0, c7, c10, 4 @ Drain write buffer @@ -132,17 +135,18 @@ ENTRY(cpu_arm925_do_idle) mcr p15, 0, r0, c7, c0, 4 @ Wait for interrupt mcr p15, 0, r1, c1, c0, 0 @ Restore ICache enable ret lr +SYM_FUNC_END(cpu_arm925_do_idle) /* * flush_icache_all() * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(arm925_flush_icache_all) +SYM_TYPED_FUNC_START(arm925_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(arm925_flush_icache_all) +SYM_FUNC_END(arm925_flush_icache_all) /* * flush_user_cache_all() @@ -150,15 +154,14 @@ ENDPROC(arm925_flush_icache_all) * Clean and invalidate all cache entries in a particular * address space. */ -ENTRY(arm925_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(arm925_flush_user_cache_all, arm925_flush_kern_cache_all) /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(arm925_flush_kern_cache_all) +SYM_TYPED_FUNC_START(arm925_flush_kern_cache_all) mov r2, #VM_EXEC mov ip, #0 __flush_whole_cache: @@ -175,6 +178,7 @@ __flush_whole_cache: mcrne p15, 0, ip, c7, c5, 0 @ invalidate I cache mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm925_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -186,7 +190,7 @@ __flush_whole_cache: * - end - end address (exclusive) * - flags - vm_flags describing address space */ -ENTRY(arm925_flush_user_cache_range) +SYM_TYPED_FUNC_START(arm925_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT @@ -212,6 +216,7 @@ ENTRY(arm925_flush_user_cache_range) tst r2, #VM_EXEC mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm925_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -223,8 +228,11 @@ ENTRY(arm925_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm925_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(arm925_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b arm925_coherent_user_range +#endif +SYM_FUNC_END(arm925_coherent_kern_range) /* * coherent_user_range(start, end) @@ -236,7 +244,7 @@ ENTRY(arm925_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm925_coherent_user_range) +SYM_TYPED_FUNC_START(arm925_coherent_user_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry mcr p15, 0, r0, c7, c5, 1 @ invalidate I entry @@ -246,6 +254,7 @@ ENTRY(arm925_coherent_user_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov r0, #0 ret lr +SYM_FUNC_END(arm925_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -256,7 +265,7 @@ ENTRY(arm925_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(arm925_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(arm925_flush_kern_dcache_area) add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE @@ -266,6 +275,7 @@ ENTRY(arm925_flush_kern_dcache_area) mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm925_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -324,7 +334,7 @@ arm925_dma_clean_range: * - start - virtual start address * - end - virtual end address */ -ENTRY(arm925_dma_flush_range) +SYM_TYPED_FUNC_START(arm925_dma_flush_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH @@ -337,6 +347,7 @@ ENTRY(arm925_dma_flush_range) blo 1b mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm925_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -344,13 +355,13 @@ ENTRY(arm925_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(arm925_dma_map_area) +SYM_TYPED_FUNC_START(arm925_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq arm925_dma_clean_range bcs arm925_dma_inv_range b arm925_dma_flush_range -ENDPROC(arm925_dma_map_area) +SYM_FUNC_END(arm925_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -358,17 +369,11 @@ ENDPROC(arm925_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(arm925_dma_unmap_area) +SYM_TYPED_FUNC_START(arm925_dma_unmap_area) ret lr -ENDPROC(arm925_dma_unmap_area) - - .globl arm925_flush_kern_cache_louis - .equ arm925_flush_kern_cache_louis, arm925_flush_kern_cache_all - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions arm925 +SYM_FUNC_END(arm925_dma_unmap_area) -ENTRY(cpu_arm925_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_arm925_dcache_clean_area) #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHE_DLINESIZE @@ -377,6 +382,7 @@ ENTRY(cpu_arm925_dcache_clean_area) #endif mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(cpu_arm925_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -388,7 +394,7 @@ ENTRY(cpu_arm925_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_arm925_switch_mm) +SYM_TYPED_FUNC_START(cpu_arm925_switch_mm) #ifdef CONFIG_MMU mov ip, #0 #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH @@ -406,6 +412,7 @@ ENTRY(cpu_arm925_switch_mm) mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs #endif ret lr +SYM_FUNC_END(cpu_arm925_switch_mm) /* * cpu_arm925_set_pte_ext(ptep, pte, ext) @@ -413,7 +420,7 @@ ENTRY(cpu_arm925_switch_mm) * Set a PTE and flush it out */ .align 5 -ENTRY(cpu_arm925_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_arm925_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext mov r0, r0 @@ -423,6 +430,7 @@ ENTRY(cpu_arm925_set_pte_ext) mcr p15, 0, r0, c7, c10, 4 @ drain WB #endif /* CONFIG_MMU */ ret lr +SYM_FUNC_END(cpu_arm925_set_pte_ext) .type __arm925_setup, #function __arm925_setup: diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S index 1ba253c2bce1..6f43d6af2d9a 100644 --- a/arch/arm/mm/proc-arm926.S +++ b/arch/arm/mm/proc-arm926.S @@ -13,10 +13,11 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/page.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -40,18 +41,20 @@ /* * cpu_arm926_proc_init() */ -ENTRY(cpu_arm926_proc_init) +SYM_TYPED_FUNC_START(cpu_arm926_proc_init) ret lr +SYM_FUNC_END(cpu_arm926_proc_init) /* * cpu_arm926_proc_fin() */ -ENTRY(cpu_arm926_proc_fin) +SYM_TYPED_FUNC_START(cpu_arm926_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1000 @ ...i............ bic r0, r0, #0x000e @ ............wca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_arm926_proc_fin) /* * cpu_arm926_reset(loc) @@ -64,7 +67,7 @@ ENTRY(cpu_arm926_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_arm926_reset) +SYM_TYPED_FUNC_START(cpu_arm926_reset) mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I,D caches mcr p15, 0, ip, c7, c10, 4 @ drain WB @@ -76,7 +79,7 @@ ENTRY(cpu_arm926_reset) bic ip, ip, #0x1100 @ ...i...s........ mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_arm926_reset) +SYM_FUNC_END(cpu_arm926_reset) .popsection /* @@ -85,7 +88,7 @@ ENDPROC(cpu_arm926_reset) * Called with IRQs disabled */ .align 10 -ENTRY(cpu_arm926_do_idle) +SYM_TYPED_FUNC_START(cpu_arm926_do_idle) mov r0, #0 mrc p15, 0, r1, c1, c0, 0 @ Read control register mcr p15, 0, r0, c7, c10, 4 @ Drain write buffer @@ -98,17 +101,18 @@ ENTRY(cpu_arm926_do_idle) mcr p15, 0, r1, c1, c0, 0 @ Restore ICache enable msr cpsr_c, r3 @ Restore FIQ state ret lr +SYM_FUNC_END(cpu_arm926_do_idle) /* * flush_icache_all() * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(arm926_flush_icache_all) +SYM_TYPED_FUNC_START(arm926_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(arm926_flush_icache_all) +SYM_FUNC_END(arm926_flush_icache_all) /* * flush_user_cache_all() @@ -116,15 +120,14 @@ ENDPROC(arm926_flush_icache_all) * Clean and invalidate all cache entries in a particular * address space. */ -ENTRY(arm926_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(arm926_flush_user_cache_all, arm926_flush_kern_cache_all) /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(arm926_flush_kern_cache_all) +SYM_TYPED_FUNC_START(arm926_flush_kern_cache_all) mov r2, #VM_EXEC mov ip, #0 __flush_whole_cache: @@ -138,6 +141,7 @@ __flush_whole_cache: mcrne p15, 0, ip, c7, c5, 0 @ invalidate I cache mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm926_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -149,7 +153,7 @@ __flush_whole_cache: * - end - end address (exclusive) * - flags - vm_flags describing address space */ -ENTRY(arm926_flush_user_cache_range) +SYM_TYPED_FUNC_START(arm926_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT @@ -175,6 +179,7 @@ ENTRY(arm926_flush_user_cache_range) tst r2, #VM_EXEC mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm926_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -186,8 +191,11 @@ ENTRY(arm926_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm926_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(arm926_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b arm926_coherent_user_range +#endif +SYM_FUNC_END(arm926_coherent_kern_range) /* * coherent_user_range(start, end) @@ -199,7 +207,7 @@ ENTRY(arm926_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm926_coherent_user_range) +SYM_TYPED_FUNC_START(arm926_coherent_user_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry mcr p15, 0, r0, c7, c5, 1 @ invalidate I entry @@ -209,6 +217,7 @@ ENTRY(arm926_coherent_user_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov r0, #0 ret lr +SYM_FUNC_END(arm926_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -219,7 +228,7 @@ ENTRY(arm926_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(arm926_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(arm926_flush_kern_dcache_area) add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE @@ -229,6 +238,7 @@ ENTRY(arm926_flush_kern_dcache_area) mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm926_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -287,7 +297,7 @@ arm926_dma_clean_range: * - start - virtual start address * - end - virtual end address */ -ENTRY(arm926_dma_flush_range) +SYM_TYPED_FUNC_START(arm926_dma_flush_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH @@ -300,6 +310,7 @@ ENTRY(arm926_dma_flush_range) blo 1b mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm926_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -307,13 +318,13 @@ ENTRY(arm926_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(arm926_dma_map_area) +SYM_TYPED_FUNC_START(arm926_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq arm926_dma_clean_range bcs arm926_dma_inv_range b arm926_dma_flush_range -ENDPROC(arm926_dma_map_area) +SYM_FUNC_END(arm926_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -321,17 +332,11 @@ ENDPROC(arm926_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(arm926_dma_unmap_area) +SYM_TYPED_FUNC_START(arm926_dma_unmap_area) ret lr -ENDPROC(arm926_dma_unmap_area) - - .globl arm926_flush_kern_cache_louis - .equ arm926_flush_kern_cache_louis, arm926_flush_kern_cache_all +SYM_FUNC_END(arm926_dma_unmap_area) - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions arm926 - -ENTRY(cpu_arm926_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_arm926_dcache_clean_area) #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHE_DLINESIZE @@ -340,6 +345,7 @@ ENTRY(cpu_arm926_dcache_clean_area) #endif mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(cpu_arm926_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -351,7 +357,8 @@ ENTRY(cpu_arm926_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_arm926_switch_mm) + +SYM_TYPED_FUNC_START(cpu_arm926_switch_mm) #ifdef CONFIG_MMU mov ip, #0 #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH @@ -367,6 +374,7 @@ ENTRY(cpu_arm926_switch_mm) mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs #endif ret lr +SYM_FUNC_END(cpu_arm926_switch_mm) /* * cpu_arm926_set_pte_ext(ptep, pte, ext) @@ -374,7 +382,7 @@ ENTRY(cpu_arm926_switch_mm) * Set a PTE and flush it out */ .align 5 -ENTRY(cpu_arm926_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_arm926_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext mov r0, r0 @@ -384,21 +392,22 @@ ENTRY(cpu_arm926_set_pte_ext) mcr p15, 0, r0, c7, c10, 4 @ drain WB #endif ret lr +SYM_FUNC_END(cpu_arm926_set_pte_ext) /* Suspend/resume support: taken from arch/arm/plat-s3c24xx/sleep.S */ .globl cpu_arm926_suspend_size .equ cpu_arm926_suspend_size, 4 * 3 #ifdef CONFIG_ARM_CPU_SUSPEND -ENTRY(cpu_arm926_do_suspend) +SYM_TYPED_FUNC_START(cpu_arm926_do_suspend) stmfd sp!, {r4 - r6, lr} mrc p15, 0, r4, c13, c0, 0 @ PID mrc p15, 0, r5, c3, c0, 0 @ Domain ID mrc p15, 0, r6, c1, c0, 0 @ Control register stmia r0, {r4 - r6} ldmfd sp!, {r4 - r6, pc} -ENDPROC(cpu_arm926_do_suspend) +SYM_FUNC_END(cpu_arm926_do_suspend) -ENTRY(cpu_arm926_do_resume) +SYM_TYPED_FUNC_START(cpu_arm926_do_resume) mov ip, #0 mcr p15, 0, ip, c8, c7, 0 @ invalidate I+D TLBs mcr p15, 0, ip, c7, c7, 0 @ invalidate I+D caches @@ -408,7 +417,7 @@ ENTRY(cpu_arm926_do_resume) mcr p15, 0, r1, c2, c0, 0 @ TTB address mov r0, r6 @ control register b cpu_resume_mmu -ENDPROC(cpu_arm926_do_resume) +SYM_FUNC_END(cpu_arm926_do_resume) #endif .type __arm926_setup, #function diff --git a/arch/arm/mm/proc-arm940.S b/arch/arm/mm/proc-arm940.S index 4b8a00220cc9..0d30bb25c42b 100644 --- a/arch/arm/mm/proc-arm940.S +++ b/arch/arm/mm/proc-arm940.S @@ -6,10 +6,11 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -25,19 +26,24 @@ * * These are not required. */ -ENTRY(cpu_arm940_proc_init) -ENTRY(cpu_arm940_switch_mm) +SYM_TYPED_FUNC_START(cpu_arm940_proc_init) ret lr +SYM_FUNC_END(cpu_arm940_proc_init) + +SYM_TYPED_FUNC_START(cpu_arm940_switch_mm) + ret lr +SYM_FUNC_END(cpu_arm940_switch_mm) /* * cpu_arm940_proc_fin() */ -ENTRY(cpu_arm940_proc_fin) +SYM_TYPED_FUNC_START(cpu_arm940_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x00001000 @ i-cache bic r0, r0, #0x00000004 @ d-cache mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_arm940_proc_fin) /* * cpu_arm940_reset(loc) @@ -45,7 +51,7 @@ ENTRY(cpu_arm940_proc_fin) * Notes : This sets up everything for a reset */ .pushsection .idmap.text, "ax" -ENTRY(cpu_arm940_reset) +SYM_TYPED_FUNC_START(cpu_arm940_reset) mov ip, #0 mcr p15, 0, ip, c7, c5, 0 @ flush I cache mcr p15, 0, ip, c7, c6, 0 @ flush D cache @@ -55,42 +61,43 @@ ENTRY(cpu_arm940_reset) bic ip, ip, #0x00001000 @ i-cache mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_arm940_reset) +SYM_FUNC_END(cpu_arm940_reset) .popsection /* * cpu_arm940_do_idle() */ .align 5 -ENTRY(cpu_arm940_do_idle) +SYM_TYPED_FUNC_START(cpu_arm940_do_idle) mcr p15, 0, r0, c7, c0, 4 @ Wait for interrupt ret lr +SYM_FUNC_END(cpu_arm940_do_idle) /* * flush_icache_all() * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(arm940_flush_icache_all) +SYM_TYPED_FUNC_START(arm940_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(arm940_flush_icache_all) +SYM_FUNC_END(arm940_flush_icache_all) /* * flush_user_cache_all() */ -ENTRY(arm940_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(arm940_flush_user_cache_all, arm940_flush_kern_cache_all) /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(arm940_flush_kern_cache_all) +SYM_TYPED_FUNC_START(arm940_flush_kern_cache_all) mov r2, #VM_EXEC - /* FALLTHROUGH */ + b arm940_flush_user_cache_range +SYM_FUNC_END(arm940_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -102,7 +109,7 @@ ENTRY(arm940_flush_kern_cache_all) * - end - end address (exclusive) * - flags - vm_flags describing address space */ -ENTRY(arm940_flush_user_cache_range) +SYM_TYPED_FUNC_START(arm940_flush_user_cache_range) mov ip, #0 #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH mcr p15, 0, ip, c7, c6, 0 @ flush D cache @@ -119,6 +126,7 @@ ENTRY(arm940_flush_user_cache_range) mcrne p15, 0, ip, c7, c5, 0 @ invalidate I cache mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm940_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -130,8 +138,9 @@ ENTRY(arm940_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm940_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(arm940_coherent_kern_range) + b arm940_flush_kern_dcache_area +SYM_FUNC_END(arm940_coherent_kern_range) /* * coherent_user_range(start, end) @@ -143,8 +152,11 @@ ENTRY(arm940_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm940_coherent_user_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(arm940_coherent_user_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b arm940_flush_kern_dcache_area +#endif +SYM_FUNC_END(arm940_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -155,7 +167,7 @@ ENTRY(arm940_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(arm940_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(arm940_flush_kern_dcache_area) mov r0, #0 mov r1, #(CACHE_DSEGMENTS - 1) << 4 @ 4 segments 1: orr r3, r1, #(CACHE_DENTRIES - 1) << 26 @ 64 entries @@ -167,6 +179,7 @@ ENTRY(arm940_flush_kern_dcache_area) mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm940_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -199,7 +212,7 @@ arm940_dma_inv_range: * - end - virtual end address */ arm940_dma_clean_range: -ENTRY(cpu_arm940_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_arm940_dcache_clean_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH mov r1, #(CACHE_DSEGMENTS - 1) << 4 @ 4 segments @@ -212,6 +225,7 @@ ENTRY(cpu_arm940_dcache_clean_area) #endif mcr p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(cpu_arm940_dcache_clean_area) /* * dma_flush_range(start, end) @@ -222,7 +236,7 @@ ENTRY(cpu_arm940_dcache_clean_area) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm940_dma_flush_range) +SYM_TYPED_FUNC_START(arm940_dma_flush_range) mov ip, #0 mov r1, #(CACHE_DSEGMENTS - 1) << 4 @ 4 segments 1: orr r3, r1, #(CACHE_DENTRIES - 1) << 26 @ 64 entries @@ -238,6 +252,7 @@ ENTRY(arm940_dma_flush_range) bcs 1b @ segments 7 to 0 mcr p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm940_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -245,13 +260,13 @@ ENTRY(arm940_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(arm940_dma_map_area) +SYM_TYPED_FUNC_START(arm940_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq arm940_dma_clean_range bcs arm940_dma_inv_range b arm940_dma_flush_range -ENDPROC(arm940_dma_map_area) +SYM_FUNC_END(arm940_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -259,15 +274,9 @@ ENDPROC(arm940_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(arm940_dma_unmap_area) +SYM_TYPED_FUNC_START(arm940_dma_unmap_area) ret lr -ENDPROC(arm940_dma_unmap_area) - - .globl arm940_flush_kern_cache_louis - .equ arm940_flush_kern_cache_louis, arm940_flush_kern_cache_all - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions arm940 +SYM_FUNC_END(arm940_dma_unmap_area) .type __arm940_setup, #function __arm940_setup: diff --git a/arch/arm/mm/proc-arm946.S b/arch/arm/mm/proc-arm946.S index 555becf9c758..27750ace2ced 100644 --- a/arch/arm/mm/proc-arm946.S +++ b/arch/arm/mm/proc-arm946.S @@ -8,10 +8,11 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -32,19 +33,24 @@ * * These are not required. */ -ENTRY(cpu_arm946_proc_init) -ENTRY(cpu_arm946_switch_mm) +SYM_TYPED_FUNC_START(cpu_arm946_proc_init) ret lr +SYM_FUNC_END(cpu_arm946_proc_init) + +SYM_TYPED_FUNC_START(cpu_arm946_switch_mm) + ret lr +SYM_FUNC_END(cpu_arm946_switch_mm) /* * cpu_arm946_proc_fin() */ -ENTRY(cpu_arm946_proc_fin) +SYM_TYPED_FUNC_START(cpu_arm946_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x00001000 @ i-cache bic r0, r0, #0x00000004 @ d-cache mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_arm946_proc_fin) /* * cpu_arm946_reset(loc) @@ -52,7 +58,7 @@ ENTRY(cpu_arm946_proc_fin) * Notes : This sets up everything for a reset */ .pushsection .idmap.text, "ax" -ENTRY(cpu_arm946_reset) +SYM_TYPED_FUNC_START(cpu_arm946_reset) mov ip, #0 mcr p15, 0, ip, c7, c5, 0 @ flush I cache mcr p15, 0, ip, c7, c6, 0 @ flush D cache @@ -62,40 +68,40 @@ ENTRY(cpu_arm946_reset) bic ip, ip, #0x00001000 @ i-cache mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_arm946_reset) +SYM_FUNC_END(cpu_arm946_reset) .popsection /* * cpu_arm946_do_idle() */ .align 5 -ENTRY(cpu_arm946_do_idle) +SYM_TYPED_FUNC_START(cpu_arm946_do_idle) mcr p15, 0, r0, c7, c0, 4 @ Wait for interrupt ret lr +SYM_FUNC_END(cpu_arm946_do_idle) /* * flush_icache_all() * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(arm946_flush_icache_all) +SYM_TYPED_FUNC_START(arm946_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(arm946_flush_icache_all) +SYM_FUNC_END(arm946_flush_icache_all) /* * flush_user_cache_all() */ -ENTRY(arm946_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(arm946_flush_user_cache_all, arm946_flush_kern_cache_all) /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(arm946_flush_kern_cache_all) +SYM_TYPED_FUNC_START(arm946_flush_kern_cache_all) mov r2, #VM_EXEC mov ip, #0 __flush_whole_cache: @@ -114,6 +120,7 @@ __flush_whole_cache: mcrne p15, 0, ip, c7, c5, 0 @ flush I cache mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm946_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -126,7 +133,7 @@ __flush_whole_cache: * - flags - vm_flags describing address space * (same as arm926) */ -ENTRY(arm946_flush_user_cache_range) +SYM_TYPED_FUNC_START(arm946_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT @@ -153,6 +160,7 @@ ENTRY(arm946_flush_user_cache_range) tst r2, #VM_EXEC mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm946_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -164,8 +172,11 @@ ENTRY(arm946_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm946_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(arm946_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b arm946_coherent_user_range +#endif +SYM_FUNC_END(arm946_coherent_kern_range) /* * coherent_user_range(start, end) @@ -178,7 +189,7 @@ ENTRY(arm946_coherent_kern_range) * - end - virtual end address * (same as arm926) */ -ENTRY(arm946_coherent_user_range) +SYM_TYPED_FUNC_START(arm946_coherent_user_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry mcr p15, 0, r0, c7, c5, 1 @ invalidate I entry @@ -188,6 +199,7 @@ ENTRY(arm946_coherent_user_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov r0, #0 ret lr +SYM_FUNC_END(arm946_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -199,7 +211,7 @@ ENTRY(arm946_coherent_user_range) * - size - region size * (same as arm926) */ -ENTRY(arm946_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(arm946_flush_kern_dcache_area) add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE @@ -209,6 +221,7 @@ ENTRY(arm946_flush_kern_dcache_area) mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm946_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -268,7 +281,7 @@ arm946_dma_clean_range: * * (same as arm926) */ -ENTRY(arm946_dma_flush_range) +SYM_TYPED_FUNC_START(arm946_dma_flush_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH @@ -281,6 +294,7 @@ ENTRY(arm946_dma_flush_range) blo 1b mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(arm946_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -288,13 +302,13 @@ ENTRY(arm946_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(arm946_dma_map_area) +SYM_TYPED_FUNC_START(arm946_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq arm946_dma_clean_range bcs arm946_dma_inv_range b arm946_dma_flush_range -ENDPROC(arm946_dma_map_area) +SYM_FUNC_END(arm946_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -302,17 +316,11 @@ ENDPROC(arm946_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(arm946_dma_unmap_area) +SYM_TYPED_FUNC_START(arm946_dma_unmap_area) ret lr -ENDPROC(arm946_dma_unmap_area) - - .globl arm946_flush_kern_cache_louis - .equ arm946_flush_kern_cache_louis, arm946_flush_kern_cache_all - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions arm946 +SYM_FUNC_END(arm946_dma_unmap_area) -ENTRY(cpu_arm946_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_arm946_dcache_clean_area) #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHE_DLINESIZE @@ -321,6 +329,7 @@ ENTRY(cpu_arm946_dcache_clean_area) #endif mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(cpu_arm946_dcache_clean_area) .type __arm946_setup, #function __arm946_setup: diff --git a/arch/arm/mm/proc-arm9tdmi.S b/arch/arm/mm/proc-arm9tdmi.S index ef517530130b..c480a8400eff 100644 --- a/arch/arm/mm/proc-arm9tdmi.S +++ b/arch/arm/mm/proc-arm9tdmi.S @@ -6,11 +6,12 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -24,17 +25,28 @@ * * These are not required. */ -ENTRY(cpu_arm9tdmi_proc_init) -ENTRY(cpu_arm9tdmi_do_idle) -ENTRY(cpu_arm9tdmi_dcache_clean_area) -ENTRY(cpu_arm9tdmi_switch_mm) +SYM_TYPED_FUNC_START(cpu_arm9tdmi_proc_init) ret lr +SYM_FUNC_END(cpu_arm9tdmi_proc_init) + +SYM_TYPED_FUNC_START(cpu_arm9tdmi_do_idle) + ret lr +SYM_FUNC_END(cpu_arm9tdmi_do_idle) + +SYM_TYPED_FUNC_START(cpu_arm9tdmi_dcache_clean_area) + ret lr +SYM_FUNC_END(cpu_arm9tdmi_dcache_clean_area) + +SYM_TYPED_FUNC_START(cpu_arm9tdmi_switch_mm) + ret lr +SYM_FUNC_END(cpu_arm9tdmi_switch_mm) /* * cpu_arm9tdmi_proc_fin() */ -ENTRY(cpu_arm9tdmi_proc_fin) +SYM_TYPED_FUNC_START(cpu_arm9tdmi_proc_fin) ret lr +SYM_FUNC_END(cpu_arm9tdmi_proc_fin) /* * Function: cpu_arm9tdmi_reset(loc) @@ -42,9 +54,9 @@ ENTRY(cpu_arm9tdmi_proc_fin) * Purpose : Sets up everything for a reset and jump to the location for soft reset. */ .pushsection .idmap.text, "ax" -ENTRY(cpu_arm9tdmi_reset) +SYM_TYPED_FUNC_START(cpu_arm9tdmi_reset) ret r0 -ENDPROC(cpu_arm9tdmi_reset) +SYM_FUNC_END(cpu_arm9tdmi_reset) .popsection .type __arm9tdmi_setup, #function diff --git a/arch/arm/mm/proc-fa526.S b/arch/arm/mm/proc-fa526.S index dddf833fe000..7c16ccac8a05 100644 --- a/arch/arm/mm/proc-fa526.S +++ b/arch/arm/mm/proc-fa526.S @@ -11,10 +11,11 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/page.h> #include <asm/ptrace.h> @@ -26,13 +27,14 @@ /* * cpu_fa526_proc_init() */ -ENTRY(cpu_fa526_proc_init) +SYM_TYPED_FUNC_START(cpu_fa526_proc_init) ret lr +SYM_FUNC_END(cpu_fa526_proc_init) /* * cpu_fa526_proc_fin() */ -ENTRY(cpu_fa526_proc_fin) +SYM_TYPED_FUNC_START(cpu_fa526_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1000 @ ...i............ bic r0, r0, #0x000e @ ............wca. @@ -40,6 +42,7 @@ ENTRY(cpu_fa526_proc_fin) nop nop ret lr +SYM_FUNC_END(cpu_fa526_proc_fin) /* * cpu_fa526_reset(loc) @@ -52,7 +55,7 @@ ENTRY(cpu_fa526_proc_fin) */ .align 4 .pushsection .idmap.text, "ax" -ENTRY(cpu_fa526_reset) +SYM_TYPED_FUNC_START(cpu_fa526_reset) /* TODO: Use CP8 if possible... */ mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I,D caches @@ -68,24 +71,25 @@ ENTRY(cpu_fa526_reset) nop nop ret r0 -ENDPROC(cpu_fa526_reset) +SYM_FUNC_END(cpu_fa526_reset) .popsection /* * cpu_fa526_do_idle() */ .align 4 -ENTRY(cpu_fa526_do_idle) +SYM_TYPED_FUNC_START(cpu_fa526_do_idle) ret lr +SYM_FUNC_END(cpu_fa526_do_idle) - -ENTRY(cpu_fa526_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_fa526_dcache_clean_area) 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHE_DLINESIZE subs r1, r1, #CACHE_DLINESIZE bhi 1b mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(cpu_fa526_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -97,7 +101,7 @@ ENTRY(cpu_fa526_dcache_clean_area) * pgd: new page tables */ .align 4 -ENTRY(cpu_fa526_switch_mm) +SYM_TYPED_FUNC_START(cpu_fa526_switch_mm) #ifdef CONFIG_MMU mov ip, #0 #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH @@ -113,6 +117,7 @@ ENTRY(cpu_fa526_switch_mm) mcr p15, 0, ip, c8, c7, 0 @ invalidate UTLB #endif ret lr +SYM_FUNC_END(cpu_fa526_switch_mm) /* * cpu_fa526_set_pte_ext(ptep, pte, ext) @@ -120,7 +125,7 @@ ENTRY(cpu_fa526_switch_mm) * Set a PTE and flush it out */ .align 4 -ENTRY(cpu_fa526_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_fa526_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext mov r0, r0 @@ -129,6 +134,7 @@ ENTRY(cpu_fa526_set_pte_ext) mcr p15, 0, r0, c7, c10, 4 @ drain WB #endif ret lr +SYM_FUNC_END(cpu_fa526_set_pte_ext) .type __fa526_setup, #function __fa526_setup: diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S index b12b76bc8d30..f67b2ffac854 100644 --- a/arch/arm/mm/proc-feroceon.S +++ b/arch/arm/mm/proc-feroceon.S @@ -8,10 +8,11 @@ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/page.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -43,7 +44,7 @@ __cache_params: /* * cpu_feroceon_proc_init() */ -ENTRY(cpu_feroceon_proc_init) +SYM_TYPED_FUNC_START(cpu_feroceon_proc_init) mrc p15, 0, r0, c0, c0, 1 @ read cache type register ldr r1, __cache_params mov r2, #(16 << 5) @@ -56,12 +57,17 @@ ENTRY(cpu_feroceon_proc_init) movne r2, r2, lsr #2 @ turned into # of sets sub r2, r2, #(1 << 5) stmia r1, {r2, r3} +#ifdef CONFIG_VFP + mov r1, #1 @ disable quirky VFP + str_l r1, VFP_arch_feroceon, r2 +#endif ret lr +SYM_FUNC_END(cpu_feroceon_proc_init) /* * cpu_feroceon_proc_fin() */ -ENTRY(cpu_feroceon_proc_fin) +SYM_TYPED_FUNC_START(cpu_feroceon_proc_fin) #if defined(CONFIG_CACHE_FEROCEON_L2) && \ !defined(CONFIG_CACHE_FEROCEON_L2_WRITETHROUGH) mov r0, #0 @@ -74,6 +80,7 @@ ENTRY(cpu_feroceon_proc_fin) bic r0, r0, #0x000e @ ............wca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_feroceon_proc_fin) /* * cpu_feroceon_reset(loc) @@ -86,7 +93,7 @@ ENTRY(cpu_feroceon_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_feroceon_reset) +SYM_TYPED_FUNC_START(cpu_feroceon_reset) mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I,D caches mcr p15, 0, ip, c7, c10, 4 @ drain WB @@ -98,7 +105,7 @@ ENTRY(cpu_feroceon_reset) bic ip, ip, #0x1100 @ ...i...s........ mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_feroceon_reset) +SYM_FUNC_END(cpu_feroceon_reset) .popsection /* @@ -107,22 +114,23 @@ ENDPROC(cpu_feroceon_reset) * Called with IRQs disabled */ .align 5 -ENTRY(cpu_feroceon_do_idle) +SYM_TYPED_FUNC_START(cpu_feroceon_do_idle) mov r0, #0 mcr p15, 0, r0, c7, c10, 4 @ Drain write buffer mcr p15, 0, r0, c7, c0, 4 @ Wait for interrupt ret lr +SYM_FUNC_END(cpu_feroceon_do_idle) /* * flush_icache_all() * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(feroceon_flush_icache_all) +SYM_TYPED_FUNC_START(feroceon_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(feroceon_flush_icache_all) +SYM_FUNC_END(feroceon_flush_icache_all) /* * flush_user_cache_all() @@ -131,15 +139,14 @@ ENDPROC(feroceon_flush_icache_all) * address space. */ .align 5 -ENTRY(feroceon_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(feroceon_flush_user_cache_all, feroceon_flush_kern_cache_all) /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(feroceon_flush_kern_cache_all) +SYM_TYPED_FUNC_START(feroceon_flush_kern_cache_all) mov r2, #VM_EXEC __flush_whole_cache: @@ -157,6 +164,7 @@ __flush_whole_cache: mcrne p15, 0, ip, c7, c5, 0 @ invalidate I cache mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(feroceon_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -169,7 +177,7 @@ __flush_whole_cache: * - flags - vm_flags describing address space */ .align 5 -ENTRY(feroceon_flush_user_cache_range) +SYM_TYPED_FUNC_START(feroceon_flush_user_cache_range) sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT bgt __flush_whole_cache @@ -186,6 +194,7 @@ ENTRY(feroceon_flush_user_cache_range) mov ip, #0 mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(feroceon_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -198,8 +207,11 @@ ENTRY(feroceon_flush_user_cache_range) * - end - virtual end address */ .align 5 -ENTRY(feroceon_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(feroceon_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b feroceon_coherent_user_range +#endif +SYM_FUNC_END(feroceon_coherent_kern_range) /* * coherent_user_range(start, end) @@ -211,7 +223,7 @@ ENTRY(feroceon_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(feroceon_coherent_user_range) +SYM_TYPED_FUNC_START(feroceon_coherent_user_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry mcr p15, 0, r0, c7, c5, 1 @ invalidate I entry @@ -221,6 +233,7 @@ ENTRY(feroceon_coherent_user_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov r0, #0 ret lr +SYM_FUNC_END(feroceon_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -232,7 +245,7 @@ ENTRY(feroceon_coherent_user_range) * - size - region size */ .align 5 -ENTRY(feroceon_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(feroceon_flush_kern_dcache_area) add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE @@ -242,9 +255,10 @@ ENTRY(feroceon_flush_kern_dcache_area) mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(feroceon_flush_kern_dcache_area) .align 5 -ENTRY(feroceon_range_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(feroceon_range_flush_kern_dcache_area) mrs r2, cpsr add r1, r0, #PAGE_SZ - CACHE_DLINESIZE @ top addr is inclusive orr r3, r2, #PSR_I_BIT @@ -256,6 +270,7 @@ ENTRY(feroceon_range_flush_kern_dcache_area) mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(feroceon_range_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -342,7 +357,7 @@ feroceon_range_dma_clean_range: * - end - virtual end address */ .align 5 -ENTRY(feroceon_dma_flush_range) +SYM_TYPED_FUNC_START(feroceon_dma_flush_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE @@ -350,9 +365,10 @@ ENTRY(feroceon_dma_flush_range) blo 1b mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(feroceon_dma_flush_range) .align 5 -ENTRY(feroceon_range_dma_flush_range) +SYM_TYPED_FUNC_START(feroceon_range_dma_flush_range) mrs r2, cpsr cmp r1, r0 subne r1, r1, #1 @ top address is inclusive @@ -363,6 +379,7 @@ ENTRY(feroceon_range_dma_flush_range) msr cpsr_c, r2 @ restore interrupts mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(feroceon_range_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -370,13 +387,13 @@ ENTRY(feroceon_range_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(feroceon_dma_map_area) +SYM_TYPED_FUNC_START(feroceon_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq feroceon_dma_clean_range bcs feroceon_dma_inv_range b feroceon_dma_flush_range -ENDPROC(feroceon_dma_map_area) +SYM_FUNC_END(feroceon_dma_map_area) /* * dma_map_area(start, size, dir) @@ -384,13 +401,13 @@ ENDPROC(feroceon_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(feroceon_range_dma_map_area) +SYM_TYPED_FUNC_START(feroceon_range_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq feroceon_range_dma_clean_range bcs feroceon_range_dma_inv_range b feroceon_range_dma_flush_range -ENDPROC(feroceon_range_dma_map_area) +SYM_FUNC_END(feroceon_range_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -398,39 +415,12 @@ ENDPROC(feroceon_range_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(feroceon_dma_unmap_area) +SYM_TYPED_FUNC_START(feroceon_dma_unmap_area) ret lr -ENDPROC(feroceon_dma_unmap_area) - - .globl feroceon_flush_kern_cache_louis - .equ feroceon_flush_kern_cache_louis, feroceon_flush_kern_cache_all - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions feroceon - -.macro range_alias basename - .globl feroceon_range_\basename - .type feroceon_range_\basename , %function - .equ feroceon_range_\basename , feroceon_\basename -.endm - -/* - * Most of the cache functions are unchanged for this case. - * Export suitable alias symbols for the unchanged functions: - */ - range_alias flush_icache_all - range_alias flush_user_cache_all - range_alias flush_kern_cache_all - range_alias flush_kern_cache_louis - range_alias flush_user_cache_range - range_alias coherent_kern_range - range_alias coherent_user_range - range_alias dma_unmap_area - - define_cache_functions feroceon_range +SYM_FUNC_END(feroceon_dma_unmap_area) .align 5 -ENTRY(cpu_feroceon_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_feroceon_dcache_clean_area) #if defined(CONFIG_CACHE_FEROCEON_L2) && \ !defined(CONFIG_CACHE_FEROCEON_L2_WRITETHROUGH) mov r2, r0 @@ -449,6 +439,7 @@ ENTRY(cpu_feroceon_dcache_clean_area) #endif mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(cpu_feroceon_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -460,7 +451,7 @@ ENTRY(cpu_feroceon_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_feroceon_switch_mm) +SYM_TYPED_FUNC_START(cpu_feroceon_switch_mm) #ifdef CONFIG_MMU /* * Note: we wish to call __flush_whole_cache but we need to preserve @@ -481,6 +472,7 @@ ENTRY(cpu_feroceon_switch_mm) #else ret lr #endif +SYM_FUNC_END(cpu_feroceon_switch_mm) /* * cpu_feroceon_set_pte_ext(ptep, pte, ext) @@ -488,7 +480,7 @@ ENTRY(cpu_feroceon_switch_mm) * Set a PTE and flush it out */ .align 5 -ENTRY(cpu_feroceon_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_feroceon_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext wc_disable=0 mov r0, r0 @@ -500,21 +492,22 @@ ENTRY(cpu_feroceon_set_pte_ext) mcr p15, 0, r0, c7, c10, 4 @ drain WB #endif ret lr +SYM_FUNC_END(cpu_feroceon_set_pte_ext) /* Suspend/resume support: taken from arch/arm/mm/proc-arm926.S */ .globl cpu_feroceon_suspend_size .equ cpu_feroceon_suspend_size, 4 * 3 #ifdef CONFIG_ARM_CPU_SUSPEND -ENTRY(cpu_feroceon_do_suspend) +SYM_TYPED_FUNC_START(cpu_feroceon_do_suspend) stmfd sp!, {r4 - r6, lr} mrc p15, 0, r4, c13, c0, 0 @ PID mrc p15, 0, r5, c3, c0, 0 @ Domain ID mrc p15, 0, r6, c1, c0, 0 @ Control register stmia r0, {r4 - r6} ldmfd sp!, {r4 - r6, pc} -ENDPROC(cpu_feroceon_do_suspend) +SYM_FUNC_END(cpu_feroceon_do_suspend) -ENTRY(cpu_feroceon_do_resume) +SYM_TYPED_FUNC_START(cpu_feroceon_do_resume) mov ip, #0 mcr p15, 0, ip, c8, c7, 0 @ invalidate I+D TLBs mcr p15, 0, ip, c7, c7, 0 @ invalidate I+D caches @@ -524,7 +517,7 @@ ENTRY(cpu_feroceon_do_resume) mcr p15, 0, r1, c2, c0, 0 @ TTB address mov r0, r6 @ control register b cpu_resume_mmu -ENDPROC(cpu_feroceon_do_resume) +SYM_FUNC_END(cpu_feroceon_do_resume) #endif .type __feroceon_setup, #function diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S index 5461d589a1e2..e388c4cc0c44 100644 --- a/arch/arm/mm/proc-macros.S +++ b/arch/arm/mm/proc-macros.S @@ -6,6 +6,7 @@ * VM_EXEC */ #include <asm/asm-offsets.h> +#include <asm/pgtable.h> #include <asm/thread_info.h> #ifdef CONFIG_CPU_V7M @@ -30,9 +31,7 @@ * act_mm - get current->active_mm */ .macro act_mm, rd - bic \rd, sp, #8128 - bic \rd, \rd, #63 - ldr \rd, [\rd, #TI_TASK] + get_current \rd .if (TSK_ACTIVE_MM > IMM12_MASK) add \rd, \rd, #TSK_ACTIVE_MM & ~IMM12_MASK .endif @@ -321,38 +320,6 @@ ENTRY(\name\()_processor_functions) #endif .endm -.macro define_cache_functions name:req - .align 2 - .type \name\()_cache_fns, #object -ENTRY(\name\()_cache_fns) - .long \name\()_flush_icache_all - .long \name\()_flush_kern_cache_all - .long \name\()_flush_kern_cache_louis - .long \name\()_flush_user_cache_all - .long \name\()_flush_user_cache_range - .long \name\()_coherent_kern_range - .long \name\()_coherent_user_range - .long \name\()_flush_kern_dcache_area - .long \name\()_dma_map_area - .long \name\()_dma_unmap_area - .long \name\()_dma_flush_range - .size \name\()_cache_fns, . - \name\()_cache_fns -.endm - -.macro define_tlb_functions name:req, flags_up:req, flags_smp - .type \name\()_tlb_fns, #object -ENTRY(\name\()_tlb_fns) - .long \name\()_flush_user_tlb_range - .long \name\()_flush_kern_tlb_range - .ifnb \flags_smp - ALT_SMP(.long \flags_smp ) - ALT_UP(.long \flags_up ) - .else - .long \flags_up - .endif - .size \name\()_tlb_fns, . - \name\()_tlb_fns -.endm - .macro globl_equ x, y .globl \x .equ \x, \y diff --git a/arch/arm/mm/proc-mohawk.S b/arch/arm/mm/proc-mohawk.S index d47d6c5cee63..8e9f38da863a 100644 --- a/arch/arm/mm/proc-mohawk.S +++ b/arch/arm/mm/proc-mohawk.S @@ -9,10 +9,11 @@ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/page.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -31,18 +32,20 @@ /* * cpu_mohawk_proc_init() */ -ENTRY(cpu_mohawk_proc_init) +SYM_TYPED_FUNC_START(cpu_mohawk_proc_init) ret lr +SYM_FUNC_END(cpu_mohawk_proc_init) /* * cpu_mohawk_proc_fin() */ -ENTRY(cpu_mohawk_proc_fin) +SYM_TYPED_FUNC_START(cpu_mohawk_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1800 @ ...iz........... bic r0, r0, #0x0006 @ .............ca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_mohawk_proc_fin) /* * cpu_mohawk_reset(loc) @@ -57,7 +60,7 @@ ENTRY(cpu_mohawk_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_mohawk_reset) +SYM_TYPED_FUNC_START(cpu_mohawk_reset) mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I,D caches mcr p15, 0, ip, c7, c10, 4 @ drain WB @@ -67,7 +70,7 @@ ENTRY(cpu_mohawk_reset) bic ip, ip, #0x1100 @ ...i...s........ mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_mohawk_reset) +SYM_FUNC_END(cpu_mohawk_reset) .popsection /* @@ -76,22 +79,23 @@ ENDPROC(cpu_mohawk_reset) * Called with IRQs disabled */ .align 5 -ENTRY(cpu_mohawk_do_idle) +SYM_TYPED_FUNC_START(cpu_mohawk_do_idle) mov r0, #0 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer mcr p15, 0, r0, c7, c0, 4 @ wait for interrupt ret lr +SYM_FUNC_END(cpu_mohawk_do_idle) /* * flush_icache_all() * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(mohawk_flush_icache_all) +SYM_TYPED_FUNC_START(mohawk_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(mohawk_flush_icache_all) +SYM_FUNC_END(mohawk_flush_icache_all) /* * flush_user_cache_all() @@ -99,15 +103,14 @@ ENDPROC(mohawk_flush_icache_all) * Clean and invalidate all cache entries in a particular * address space. */ -ENTRY(mohawk_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(mohawk_flush_user_cache_all, mohawk_flush_kern_cache_all) /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(mohawk_flush_kern_cache_all) +SYM_TYPED_FUNC_START(mohawk_flush_kern_cache_all) mov r2, #VM_EXEC mov ip, #0 __flush_whole_cache: @@ -116,6 +119,7 @@ __flush_whole_cache: mcrne p15, 0, ip, c7, c5, 0 @ invalidate I cache mcrne p15, 0, ip, c7, c10, 0 @ drain write buffer ret lr +SYM_FUNC_END(mohawk_flush_kern_cache_all) /* * flush_user_cache_range(start, end, flags) @@ -129,7 +133,7 @@ __flush_whole_cache: * * (same as arm926) */ -ENTRY(mohawk_flush_user_cache_range) +SYM_TYPED_FUNC_START(mohawk_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #CACHE_DLIMIT @@ -146,6 +150,7 @@ ENTRY(mohawk_flush_user_cache_range) tst r2, #VM_EXEC mcrne p15, 0, ip, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(mohawk_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -157,8 +162,11 @@ ENTRY(mohawk_flush_user_cache_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(mohawk_coherent_kern_range) - /* FALLTHROUGH */ +SYM_TYPED_FUNC_START(mohawk_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b mohawk_coherent_user_range +#endif +SYM_FUNC_END(mohawk_coherent_kern_range) /* * coherent_user_range(start, end) @@ -172,7 +180,7 @@ ENTRY(mohawk_coherent_kern_range) * * (same as arm926) */ -ENTRY(mohawk_coherent_user_range) +SYM_TYPED_FUNC_START(mohawk_coherent_user_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry mcr p15, 0, r0, c7, c5, 1 @ invalidate I entry @@ -182,6 +190,7 @@ ENTRY(mohawk_coherent_user_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov r0, #0 ret lr +SYM_FUNC_END(mohawk_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -192,7 +201,7 @@ ENTRY(mohawk_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(mohawk_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(mohawk_flush_kern_dcache_area) add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE @@ -202,6 +211,7 @@ ENTRY(mohawk_flush_kern_dcache_area) mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(mohawk_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -256,7 +266,7 @@ mohawk_dma_clean_range: * - start - virtual start address * - end - virtual end address */ -ENTRY(mohawk_dma_flush_range) +SYM_TYPED_FUNC_START(mohawk_dma_flush_range) bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry @@ -265,6 +275,7 @@ ENTRY(mohawk_dma_flush_range) blo 1b mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(mohawk_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -272,13 +283,13 @@ ENTRY(mohawk_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(mohawk_dma_map_area) +SYM_TYPED_FUNC_START(mohawk_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq mohawk_dma_clean_range bcs mohawk_dma_inv_range b mohawk_dma_flush_range -ENDPROC(mohawk_dma_map_area) +SYM_FUNC_END(mohawk_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -286,23 +297,18 @@ ENDPROC(mohawk_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(mohawk_dma_unmap_area) +SYM_TYPED_FUNC_START(mohawk_dma_unmap_area) ret lr -ENDPROC(mohawk_dma_unmap_area) - - .globl mohawk_flush_kern_cache_louis - .equ mohawk_flush_kern_cache_louis, mohawk_flush_kern_cache_all - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions mohawk +SYM_FUNC_END(mohawk_dma_unmap_area) -ENTRY(cpu_mohawk_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_mohawk_dcache_clean_area) 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHE_DLINESIZE subs r1, r1, #CACHE_DLINESIZE bhi 1b mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr +SYM_FUNC_END(cpu_mohawk_dcache_clean_area) /* * cpu_mohawk_switch_mm(pgd) @@ -312,7 +318,7 @@ ENTRY(cpu_mohawk_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_mohawk_switch_mm) +SYM_TYPED_FUNC_START(cpu_mohawk_switch_mm) mov ip, #0 mcr p15, 0, ip, c7, c14, 0 @ clean & invalidate all D cache mcr p15, 0, ip, c7, c5, 0 @ invalidate I cache @@ -321,6 +327,7 @@ ENTRY(cpu_mohawk_switch_mm) mcr p15, 0, r0, c2, c0, 0 @ load page table pointer mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs ret lr +SYM_FUNC_END(cpu_mohawk_switch_mm) /* * cpu_mohawk_set_pte_ext(ptep, pte, ext) @@ -328,7 +335,7 @@ ENTRY(cpu_mohawk_switch_mm) * Set a PTE and flush it out */ .align 5 -ENTRY(cpu_mohawk_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_mohawk_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext mov r0, r0 @@ -336,11 +343,12 @@ ENTRY(cpu_mohawk_set_pte_ext) mcr p15, 0, r0, c7, c10, 4 @ drain WB ret lr #endif +SYM_FUNC_END(cpu_mohawk_set_pte_ext) .globl cpu_mohawk_suspend_size .equ cpu_mohawk_suspend_size, 4 * 6 #ifdef CONFIG_ARM_CPU_SUSPEND -ENTRY(cpu_mohawk_do_suspend) +SYM_TYPED_FUNC_START(cpu_mohawk_do_suspend) stmfd sp!, {r4 - r9, lr} mrc p14, 0, r4, c6, c0, 0 @ clock configuration, for turbo mode mrc p15, 0, r5, c15, c1, 0 @ CP access reg @@ -351,9 +359,9 @@ ENTRY(cpu_mohawk_do_suspend) bic r4, r4, #2 @ clear frequency change bit stmia r0, {r4 - r9} @ store cp regs ldmia sp!, {r4 - r9, pc} -ENDPROC(cpu_mohawk_do_suspend) +SYM_FUNC_END(cpu_mohawk_do_suspend) -ENTRY(cpu_mohawk_do_resume) +SYM_TYPED_FUNC_START(cpu_mohawk_do_resume) ldmia r0, {r4 - r9} @ load cp regs mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I & D caches, BTB @@ -369,7 +377,7 @@ ENTRY(cpu_mohawk_do_resume) mcr p15, 0, r8, c1, c0, 1 @ auxiliary control reg mov r0, r9 @ control register b cpu_resume_mmu -ENDPROC(cpu_mohawk_do_resume) +SYM_FUNC_END(cpu_mohawk_do_resume) #endif .type __mohawk_setup, #function diff --git a/arch/arm/mm/proc-sa110.S b/arch/arm/mm/proc-sa110.S index baba503ba816..3da76fab8ac3 100644 --- a/arch/arm/mm/proc-sa110.S +++ b/arch/arm/mm/proc-sa110.S @@ -12,12 +12,13 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/hwcap.h> #include <mach/hardware.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include <asm/ptrace.h> #include "proc-macros.S" @@ -32,15 +33,16 @@ /* * cpu_sa110_proc_init() */ -ENTRY(cpu_sa110_proc_init) +SYM_TYPED_FUNC_START(cpu_sa110_proc_init) mov r0, #0 mcr p15, 0, r0, c15, c1, 2 @ Enable clock switching ret lr +SYM_FUNC_END(cpu_sa110_proc_init) /* * cpu_sa110_proc_fin() */ -ENTRY(cpu_sa110_proc_fin) +SYM_TYPED_FUNC_START(cpu_sa110_proc_fin) mov r0, #0 mcr p15, 0, r0, c15, c2, 2 @ Disable clock switching mrc p15, 0, r0, c1, c0, 0 @ ctrl register @@ -48,6 +50,7 @@ ENTRY(cpu_sa110_proc_fin) bic r0, r0, #0x000e @ ............wca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_sa110_proc_fin) /* * cpu_sa110_reset(loc) @@ -60,7 +63,7 @@ ENTRY(cpu_sa110_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_sa110_reset) +SYM_TYPED_FUNC_START(cpu_sa110_reset) mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I,D caches mcr p15, 0, ip, c7, c10, 4 @ drain WB @@ -72,7 +75,7 @@ ENTRY(cpu_sa110_reset) bic ip, ip, #0x1100 @ ...i...s........ mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_sa110_reset) +SYM_FUNC_END(cpu_sa110_reset) .popsection /* @@ -88,7 +91,7 @@ ENDPROC(cpu_sa110_reset) */ .align 5 -ENTRY(cpu_sa110_do_idle) +SYM_TYPED_FUNC_START(cpu_sa110_do_idle) mcr p15, 0, ip, c15, c2, 2 @ disable clock switching ldr r1, =UNCACHEABLE_ADDR @ load from uncacheable loc ldr r1, [r1, #0] @ force switch to MCLK @@ -101,6 +104,7 @@ ENTRY(cpu_sa110_do_idle) mov r0, r0 @ safety mcr p15, 0, r0, c15, c1, 2 @ enable clock switching ret lr +SYM_FUNC_END(cpu_sa110_do_idle) /* ================================= CACHE ================================ */ @@ -113,12 +117,13 @@ ENTRY(cpu_sa110_do_idle) * addr: cache-unaligned virtual address */ .align 5 -ENTRY(cpu_sa110_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_sa110_dcache_clean_area) 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #DCACHELINESIZE subs r1, r1, #DCACHELINESIZE bhi 1b ret lr +SYM_FUNC_END(cpu_sa110_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -130,7 +135,7 @@ ENTRY(cpu_sa110_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_sa110_switch_mm) +SYM_TYPED_FUNC_START(cpu_sa110_switch_mm) #ifdef CONFIG_MMU str lr, [sp, #-4]! bl v4wb_flush_kern_cache_all @ clears IP @@ -140,6 +145,7 @@ ENTRY(cpu_sa110_switch_mm) #else ret lr #endif +SYM_FUNC_END(cpu_sa110_switch_mm) /* * cpu_sa110_set_pte_ext(ptep, pte, ext) @@ -147,7 +153,7 @@ ENTRY(cpu_sa110_switch_mm) * Set a PTE and flush it out */ .align 5 -ENTRY(cpu_sa110_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_sa110_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext wc_disable=0 mov r0, r0 @@ -155,6 +161,7 @@ ENTRY(cpu_sa110_set_pte_ext) mcr p15, 0, r0, c7, c10, 4 @ drain WB #endif ret lr +SYM_FUNC_END(cpu_sa110_set_pte_ext) .type __sa110_setup, #function __sa110_setup: diff --git a/arch/arm/mm/proc-sa1100.S b/arch/arm/mm/proc-sa1100.S index 75ebacc8e4e5..7c496195e440 100644 --- a/arch/arm/mm/proc-sa1100.S +++ b/arch/arm/mm/proc-sa1100.S @@ -17,12 +17,13 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/hwcap.h> #include <mach/hardware.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include "proc-macros.S" @@ -36,11 +37,12 @@ /* * cpu_sa1100_proc_init() */ -ENTRY(cpu_sa1100_proc_init) +SYM_TYPED_FUNC_START(cpu_sa1100_proc_init) mov r0, #0 mcr p15, 0, r0, c15, c1, 2 @ Enable clock switching mcr p15, 0, r0, c9, c0, 5 @ Allow read-buffer operations from userland ret lr +SYM_FUNC_END(cpu_sa1100_proc_init) /* * cpu_sa1100_proc_fin() @@ -49,13 +51,14 @@ ENTRY(cpu_sa1100_proc_init) * - Disable interrupts * - Clean and turn off caches. */ -ENTRY(cpu_sa1100_proc_fin) +SYM_TYPED_FUNC_START(cpu_sa1100_proc_fin) mcr p15, 0, ip, c15, c2, 2 @ Disable clock switching mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1000 @ ...i............ bic r0, r0, #0x000e @ ............wca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_sa1100_proc_fin) /* * cpu_sa1100_reset(loc) @@ -68,7 +71,7 @@ ENTRY(cpu_sa1100_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_sa1100_reset) +SYM_TYPED_FUNC_START(cpu_sa1100_reset) mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I,D caches mcr p15, 0, ip, c7, c10, 4 @ drain WB @@ -80,7 +83,7 @@ ENTRY(cpu_sa1100_reset) bic ip, ip, #0x1100 @ ...i...s........ mcr p15, 0, ip, c1, c0, 0 @ ctrl register ret r0 -ENDPROC(cpu_sa1100_reset) +SYM_FUNC_END(cpu_sa1100_reset) .popsection /* @@ -95,7 +98,7 @@ ENDPROC(cpu_sa1100_reset) * 3 = switch to fast processor clock */ .align 5 -ENTRY(cpu_sa1100_do_idle) +SYM_TYPED_FUNC_START(cpu_sa1100_do_idle) mov r0, r0 @ 4 nop padding mov r0, r0 mov r0, r0 @@ -111,6 +114,7 @@ ENTRY(cpu_sa1100_do_idle) mov r0, r0 @ safety mcr p15, 0, r0, c15, c1, 2 @ enable clock switching ret lr +SYM_FUNC_END(cpu_sa1100_do_idle) /* ================================= CACHE ================================ */ @@ -123,12 +127,13 @@ ENTRY(cpu_sa1100_do_idle) * addr: cache-unaligned virtual address */ .align 5 -ENTRY(cpu_sa1100_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_sa1100_dcache_clean_area) 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #DCACHELINESIZE subs r1, r1, #DCACHELINESIZE bhi 1b ret lr +SYM_FUNC_END(cpu_sa1100_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -140,7 +145,7 @@ ENTRY(cpu_sa1100_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_sa1100_switch_mm) +SYM_TYPED_FUNC_START(cpu_sa1100_switch_mm) #ifdef CONFIG_MMU str lr, [sp, #-4]! bl v4wb_flush_kern_cache_all @ clears IP @@ -151,6 +156,7 @@ ENTRY(cpu_sa1100_switch_mm) #else ret lr #endif +SYM_FUNC_END(cpu_sa1100_switch_mm) /* * cpu_sa1100_set_pte_ext(ptep, pte, ext) @@ -158,7 +164,7 @@ ENTRY(cpu_sa1100_switch_mm) * Set a PTE and flush it out */ .align 5 -ENTRY(cpu_sa1100_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_sa1100_set_pte_ext) #ifdef CONFIG_MMU armv3_set_pte_ext wc_disable=0 mov r0, r0 @@ -166,20 +172,21 @@ ENTRY(cpu_sa1100_set_pte_ext) mcr p15, 0, r0, c7, c10, 4 @ drain WB #endif ret lr +SYM_FUNC_END(cpu_sa1100_set_pte_ext) .globl cpu_sa1100_suspend_size .equ cpu_sa1100_suspend_size, 4 * 3 #ifdef CONFIG_ARM_CPU_SUSPEND -ENTRY(cpu_sa1100_do_suspend) +SYM_TYPED_FUNC_START(cpu_sa1100_do_suspend) stmfd sp!, {r4 - r6, lr} mrc p15, 0, r4, c3, c0, 0 @ domain ID mrc p15, 0, r5, c13, c0, 0 @ PID mrc p15, 0, r6, c1, c0, 0 @ control reg stmia r0, {r4 - r6} @ store cp regs ldmfd sp!, {r4 - r6, pc} -ENDPROC(cpu_sa1100_do_suspend) +SYM_FUNC_END(cpu_sa1100_do_suspend) -ENTRY(cpu_sa1100_do_resume) +SYM_TYPED_FUNC_START(cpu_sa1100_do_resume) ldmia r0, {r4 - r6} @ load cp regs mov ip, #0 mcr p15, 0, ip, c8, c7, 0 @ flush I+D TLBs @@ -192,7 +199,7 @@ ENTRY(cpu_sa1100_do_resume) mcr p15, 0, r5, c13, c0, 0 @ PID mov r0, r6 @ control register b cpu_resume_mmu -ENDPROC(cpu_sa1100_do_resume) +SYM_FUNC_END(cpu_sa1100_do_resume) #endif .type __sa1100_setup, #function diff --git a/arch/arm/mm/proc-v6.S b/arch/arm/mm/proc-v6.S index 1dd0d5ca27da..90a01f5950b9 100644 --- a/arch/arm/mm/proc-v6.S +++ b/arch/arm/mm/proc-v6.S @@ -8,12 +8,13 @@ * This is the "shell" of the ARMv6 processor support. */ #include <linux/init.h> +#include <linux/cfi_types.h> #include <linux/linkage.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> #include "proc-macros.S" @@ -32,15 +33,19 @@ #define TTB_FLAGS_SMP TTB_RGN_WBWA|TTB_S #define PMD_FLAGS_SMP PMD_SECT_WBWA|PMD_SECT_S -ENTRY(cpu_v6_proc_init) +.arch armv6 + +SYM_TYPED_FUNC_START(cpu_v6_proc_init) ret lr +SYM_FUNC_END(cpu_v6_proc_init) -ENTRY(cpu_v6_proc_fin) +SYM_TYPED_FUNC_START(cpu_v6_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1000 @ ...i............ bic r0, r0, #0x0006 @ .............ca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_v6_proc_fin) /* * cpu_v6_reset(loc) @@ -53,14 +58,14 @@ ENTRY(cpu_v6_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_v6_reset) +SYM_TYPED_FUNC_START(cpu_v6_reset) mrc p15, 0, r1, c1, c0, 0 @ ctrl register bic r1, r1, #0x1 @ ...............m mcr p15, 0, r1, c1, c0, 0 @ disable MMU mov r1, #0 mcr p15, 0, r1, c7, c5, 4 @ ISB ret r0 -ENDPROC(cpu_v6_reset) +SYM_FUNC_END(cpu_v6_reset) .popsection /* @@ -70,18 +75,20 @@ ENDPROC(cpu_v6_reset) * * IRQs are already disabled. */ -ENTRY(cpu_v6_do_idle) +SYM_TYPED_FUNC_START(cpu_v6_do_idle) mov r1, #0 mcr p15, 0, r1, c7, c10, 4 @ DWB - WFI may enter a low-power mode mcr p15, 0, r1, c7, c0, 4 @ wait for interrupt ret lr +SYM_FUNC_END(cpu_v6_do_idle) -ENTRY(cpu_v6_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_v6_dcache_clean_area) 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #D_CACHE_LINE_SIZE subs r1, r1, #D_CACHE_LINE_SIZE bhi 1b ret lr +SYM_FUNC_END(cpu_v6_dcache_clean_area) /* * cpu_v6_switch_mm(pgd_phys, tsk) @@ -93,7 +100,7 @@ ENTRY(cpu_v6_dcache_clean_area) * It is assumed that: * - we are not using split page tables */ -ENTRY(cpu_v6_switch_mm) +SYM_TYPED_FUNC_START(cpu_v6_switch_mm) #ifdef CONFIG_MMU mov r2, #0 mmid r1, r1 @ get mm->context.id @@ -111,6 +118,7 @@ ENTRY(cpu_v6_switch_mm) mcr p15, 0, r1, c13, c0, 1 @ set context ID #endif ret lr +SYM_FUNC_END(cpu_v6_switch_mm) /* * cpu_v6_set_pte_ext(ptep, pte, ext) @@ -124,17 +132,18 @@ ENTRY(cpu_v6_switch_mm) */ armv6_mt_table cpu_v6 -ENTRY(cpu_v6_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_v6_set_pte_ext) #ifdef CONFIG_MMU armv6_set_pte_ext cpu_v6 #endif ret lr +SYM_FUNC_END(cpu_v6_set_pte_ext) /* Suspend/resume support: taken from arch/arm/mach-s3c64xx/sleep.S */ .globl cpu_v6_suspend_size .equ cpu_v6_suspend_size, 4 * 6 #ifdef CONFIG_ARM_CPU_SUSPEND -ENTRY(cpu_v6_do_suspend) +SYM_TYPED_FUNC_START(cpu_v6_do_suspend) stmfd sp!, {r4 - r9, lr} mrc p15, 0, r4, c13, c0, 0 @ FCSE/PID #ifdef CONFIG_MMU @@ -146,9 +155,9 @@ ENTRY(cpu_v6_do_suspend) mrc p15, 0, r9, c1, c0, 0 @ control register stmia r0, {r4 - r9} ldmfd sp!, {r4- r9, pc} -ENDPROC(cpu_v6_do_suspend) +SYM_FUNC_END(cpu_v6_do_suspend) -ENTRY(cpu_v6_do_resume) +SYM_TYPED_FUNC_START(cpu_v6_do_resume) mov ip, #0 mcr p15, 0, ip, c7, c14, 0 @ clean+invalidate D cache mcr p15, 0, ip, c7, c5, 0 @ invalidate I cache @@ -170,7 +179,7 @@ ENTRY(cpu_v6_do_resume) mcr p15, 0, ip, c7, c5, 4 @ ISB mov r0, r9 @ control register b cpu_resume_mmu -ENDPROC(cpu_v6_do_resume) +SYM_FUNC_END(cpu_v6_do_resume) #endif string cpu_v6_name, "ARMv6-compatible processor" diff --git a/arch/arm/mm/proc-v7-2level.S b/arch/arm/mm/proc-v7-2level.S index 5db029c8f987..1007702fcaf3 100644 --- a/arch/arm/mm/proc-v7-2level.S +++ b/arch/arm/mm/proc-v7-2level.S @@ -24,6 +24,8 @@ #define TTB_FLAGS_SMP TTB_IRGN_WBWA|TTB_S|TTB_NOS|TTB_RGN_OC_WBWA #define PMD_FLAGS_SMP PMD_SECT_WBWA|PMD_SECT_S +.arch armv7-a + /* * cpu_v7_switch_mm(pgd_phys, tsk) * @@ -38,7 +40,7 @@ * even on Cortex-A8 revisions not affected by 430973. * If IBE is not set, the flush BTAC/BTB won't do anything. */ -ENTRY(cpu_v7_switch_mm) +SYM_TYPED_FUNC_START(cpu_v7_switch_mm) #ifdef CONFIG_MMU mmid r1, r1 @ get mm->context.id ALT_SMP(orr r0, r0, #TTB_FLAGS_SMP) @@ -57,7 +59,7 @@ ENTRY(cpu_v7_switch_mm) isb #endif bx lr -ENDPROC(cpu_v7_switch_mm) +SYM_FUNC_END(cpu_v7_switch_mm) /* * cpu_v7_set_pte_ext(ptep, pte) @@ -69,7 +71,7 @@ ENDPROC(cpu_v7_switch_mm) * - pte - PTE value to store * - ext - value for extended PTE bits */ -ENTRY(cpu_v7_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_v7_set_pte_ext) #ifdef CONFIG_MMU str r1, [r0] @ linux version @@ -104,7 +106,7 @@ ENTRY(cpu_v7_set_pte_ext) ALT_UP (mcr p15, 0, r0, c7, c10, 1) @ flush_pte #endif bx lr -ENDPROC(cpu_v7_set_pte_ext) +SYM_FUNC_END(cpu_v7_set_pte_ext) /* * Memory region attributes with SCTLR.TRE=1 diff --git a/arch/arm/mm/proc-v7-3level.S b/arch/arm/mm/proc-v7-3level.S index 131984462d0d..bdabc15cde56 100644 --- a/arch/arm/mm/proc-v7-3level.S +++ b/arch/arm/mm/proc-v7-3level.S @@ -42,7 +42,7 @@ * Set the translation table base pointer to be pgd_phys (physical address of * the new TTB). */ -ENTRY(cpu_v7_switch_mm) +SYM_TYPED_FUNC_START(cpu_v7_switch_mm) #ifdef CONFIG_MMU mmid r2, r2 asid r2, r2 @@ -51,7 +51,7 @@ ENTRY(cpu_v7_switch_mm) isb #endif ret lr -ENDPROC(cpu_v7_switch_mm) +SYM_FUNC_END(cpu_v7_switch_mm) #ifdef __ARMEB__ #define rl r3 @@ -68,7 +68,7 @@ ENDPROC(cpu_v7_switch_mm) * - ptep - pointer to level 3 translation table entry * - pte - PTE value to store (64-bit in r2 and r3) */ -ENTRY(cpu_v7_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_v7_set_pte_ext) #ifdef CONFIG_MMU tst rl, #L_PTE_VALID beq 1f @@ -87,7 +87,7 @@ ENTRY(cpu_v7_set_pte_ext) ALT_UP (mcr p15, 0, r0, c7, c10, 1) @ flush_pte #endif ret lr -ENDPROC(cpu_v7_set_pte_ext) +SYM_FUNC_END(cpu_v7_set_pte_ext) /* * Memory region attributes for LPAE (defined in pgtable-3level.h): diff --git a/arch/arm/mm/proc-v7-bugs.c b/arch/arm/mm/proc-v7-bugs.c index c0fbfca5da8b..8bc7a2d6d6c7 100644 --- a/arch/arm/mm/proc-v7-bugs.c +++ b/arch/arm/mm/proc-v7-bugs.c @@ -6,8 +6,35 @@ #include <asm/cp15.h> #include <asm/cputype.h> #include <asm/proc-fns.h> +#include <asm/spectre.h> #include <asm/system_misc.h> +#ifdef CONFIG_ARM_PSCI +static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, + ARM_SMCCC_ARCH_WORKAROUND_1, &res); + + switch ((int)res.a0) { + case SMCCC_RET_SUCCESS: + return SPECTRE_MITIGATED; + + case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED: + return SPECTRE_UNAFFECTED; + + default: + return SPECTRE_VULNERABLE; + } +} +#else +static int __maybe_unused spectre_v2_get_cpu_fw_mitigation_state(void) +{ + return SPECTRE_VULNERABLE; +} +#endif + #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn); @@ -36,13 +63,60 @@ static void __maybe_unused call_hvc_arch_workaround_1(void) arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); } -static void cpu_v7_spectre_init(void) +static unsigned int spectre_v2_install_workaround(unsigned int method) { const char *spectre_v2_method = NULL; int cpu = smp_processor_id(); if (per_cpu(harden_branch_predictor_fn, cpu)) - return; + return SPECTRE_MITIGATED; + + switch (method) { + case SPECTRE_V2_METHOD_BPIALL: + per_cpu(harden_branch_predictor_fn, cpu) = + harden_branch_predictor_bpiall; + spectre_v2_method = "BPIALL"; + break; + + case SPECTRE_V2_METHOD_ICIALLU: + per_cpu(harden_branch_predictor_fn, cpu) = + harden_branch_predictor_iciallu; + spectre_v2_method = "ICIALLU"; + break; + + case SPECTRE_V2_METHOD_HVC: + per_cpu(harden_branch_predictor_fn, cpu) = + call_hvc_arch_workaround_1; + cpu_do_switch_mm = cpu_v7_hvc_switch_mm; + spectre_v2_method = "hypervisor"; + break; + + case SPECTRE_V2_METHOD_SMC: + per_cpu(harden_branch_predictor_fn, cpu) = + call_smc_arch_workaround_1; + cpu_do_switch_mm = cpu_v7_smc_switch_mm; + spectre_v2_method = "firmware"; + break; + } + + if (spectre_v2_method) + pr_info("CPU%u: Spectre v2: using %s workaround\n", + smp_processor_id(), spectre_v2_method); + + return SPECTRE_MITIGATED; +} +#else +static unsigned int spectre_v2_install_workaround(unsigned int method) +{ + pr_info_once("Spectre V2: workarounds disabled by configuration\n"); + + return SPECTRE_VULNERABLE; +} +#endif + +static void cpu_v7_spectre_v2_init(void) +{ + unsigned int state, method = 0; switch (read_cpuid_part()) { case ARM_CPU_PART_CORTEX_A8: @@ -51,69 +125,133 @@ static void cpu_v7_spectre_init(void) case ARM_CPU_PART_CORTEX_A17: case ARM_CPU_PART_CORTEX_A73: case ARM_CPU_PART_CORTEX_A75: - per_cpu(harden_branch_predictor_fn, cpu) = - harden_branch_predictor_bpiall; - spectre_v2_method = "BPIALL"; + state = SPECTRE_MITIGATED; + method = SPECTRE_V2_METHOD_BPIALL; break; case ARM_CPU_PART_CORTEX_A15: case ARM_CPU_PART_BRAHMA_B15: - per_cpu(harden_branch_predictor_fn, cpu) = - harden_branch_predictor_iciallu; - spectre_v2_method = "ICIALLU"; + state = SPECTRE_MITIGATED; + method = SPECTRE_V2_METHOD_ICIALLU; break; -#ifdef CONFIG_ARM_PSCI case ARM_CPU_PART_BRAHMA_B53: /* Requires no workaround */ + state = SPECTRE_UNAFFECTED; break; + default: /* Other ARM CPUs require no workaround */ - if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) + if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) { + state = SPECTRE_UNAFFECTED; break; - /* fallthrough */ - /* Cortex A57/A72 require firmware workaround */ - case ARM_CPU_PART_CORTEX_A57: - case ARM_CPU_PART_CORTEX_A72: { - struct arm_smccc_res res; + } - arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, - ARM_SMCCC_ARCH_WORKAROUND_1, &res); - if ((int)res.a0 != 0) - return; + fallthrough; + + /* Cortex A57/A72 require firmware workaround */ + case ARM_CPU_PART_CORTEX_A57: + case ARM_CPU_PART_CORTEX_A72: + state = spectre_v2_get_cpu_fw_mitigation_state(); + if (state != SPECTRE_MITIGATED) + break; switch (arm_smccc_1_1_get_conduit()) { case SMCCC_CONDUIT_HVC: - per_cpu(harden_branch_predictor_fn, cpu) = - call_hvc_arch_workaround_1; - cpu_do_switch_mm = cpu_v7_hvc_switch_mm; - spectre_v2_method = "hypervisor"; + method = SPECTRE_V2_METHOD_HVC; break; case SMCCC_CONDUIT_SMC: - per_cpu(harden_branch_predictor_fn, cpu) = - call_smc_arch_workaround_1; - cpu_do_switch_mm = cpu_v7_smc_switch_mm; - spectre_v2_method = "firmware"; + method = SPECTRE_V2_METHOD_SMC; break; default: + state = SPECTRE_VULNERABLE; break; } } -#endif + + if (state == SPECTRE_MITIGATED) + state = spectre_v2_install_workaround(method); + + spectre_v2_update_state(state, method); +} + +#ifdef CONFIG_HARDEN_BRANCH_HISTORY +static int spectre_bhb_method; + +static const char *spectre_bhb_method_name(int method) +{ + switch (method) { + case SPECTRE_V2_METHOD_LOOP8: + return "loop"; + + case SPECTRE_V2_METHOD_BPIALL: + return "BPIALL"; + + default: + return "unknown"; } +} - if (spectre_v2_method) - pr_info("CPU%u: Spectre v2: using %s workaround\n", - smp_processor_id(), spectre_v2_method); +static int spectre_bhb_install_workaround(int method) +{ + if (spectre_bhb_method != method) { + if (spectre_bhb_method) { + pr_err("CPU%u: Spectre BHB: method disagreement, system vulnerable\n", + smp_processor_id()); + + return SPECTRE_VULNERABLE; + } + + if (spectre_bhb_update_vectors(method) == SPECTRE_VULNERABLE) + return SPECTRE_VULNERABLE; + + spectre_bhb_method = method; + + pr_info("CPU%u: Spectre BHB: enabling %s workaround for all CPUs\n", + smp_processor_id(), spectre_bhb_method_name(method)); + } + + return SPECTRE_MITIGATED; } #else -static void cpu_v7_spectre_init(void) +static int spectre_bhb_install_workaround(int method) { + return SPECTRE_VULNERABLE; } #endif +static void cpu_v7_spectre_bhb_init(void) +{ + unsigned int state, method = 0; + + switch (read_cpuid_part()) { + case ARM_CPU_PART_CORTEX_A15: + case ARM_CPU_PART_BRAHMA_B15: + case ARM_CPU_PART_CORTEX_A57: + case ARM_CPU_PART_CORTEX_A72: + state = SPECTRE_MITIGATED; + method = SPECTRE_V2_METHOD_LOOP8; + break; + + case ARM_CPU_PART_CORTEX_A73: + case ARM_CPU_PART_CORTEX_A75: + state = SPECTRE_MITIGATED; + method = SPECTRE_V2_METHOD_BPIALL; + break; + + default: + state = SPECTRE_UNAFFECTED; + break; + } + + if (state == SPECTRE_MITIGATED) + state = spectre_bhb_install_workaround(method); + + spectre_v2_update_state(state, method); +} + static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned, u32 mask, const char *msg) { @@ -142,16 +280,18 @@ static bool check_spectre_auxcr(bool *warned, u32 bit) void cpu_v7_ca8_ibe(void) { if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(6))) - cpu_v7_spectre_init(); + cpu_v7_spectre_v2_init(); } void cpu_v7_ca15_ibe(void) { if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(0))) - cpu_v7_spectre_init(); + cpu_v7_spectre_v2_init(); + cpu_v7_spectre_bhb_init(); } void cpu_v7_bugs_init(void) { - cpu_v7_spectre_init(); + cpu_v7_spectre_v2_init(); + cpu_v7_spectre_bhb_init(); } diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index 48e0ef6f0dcc..2cd933342679 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -7,14 +7,15 @@ * This is the "shell" of the ARMv7 processor support. */ #include <linux/arm-smccc.h> +#include <linux/cfi_types.h> #include <linux/init.h> #include <linux/linkage.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/hwcap.h> #include <asm/pgtable-hwdef.h> -#include <asm/pgtable.h> -#include <asm/memory.h> +#include <asm/page.h> #include "proc-macros.S" @@ -24,17 +25,19 @@ #include "proc-v7-2level.S" #endif -ENTRY(cpu_v7_proc_init) +.arch armv7-a + +SYM_TYPED_FUNC_START(cpu_v7_proc_init) ret lr -ENDPROC(cpu_v7_proc_init) +SYM_FUNC_END(cpu_v7_proc_init) -ENTRY(cpu_v7_proc_fin) +SYM_TYPED_FUNC_START(cpu_v7_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1000 @ ...i............ bic r0, r0, #0x0006 @ .............ca. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr -ENDPROC(cpu_v7_proc_fin) +SYM_FUNC_END(cpu_v7_proc_fin) /* * cpu_v7_reset(loc, hyp) @@ -51,7 +54,7 @@ ENDPROC(cpu_v7_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_v7_reset) +SYM_TYPED_FUNC_START(cpu_v7_reset) mrc p15, 0, r2, c1, c0, 0 @ ctrl register bic r2, r2, #0x1 @ ...............m THUMB( bic r2, r2, #1 << 30 ) @ SCTLR.TE (Thumb exceptions) @@ -62,7 +65,7 @@ ENTRY(cpu_v7_reset) bne __hyp_soft_restart #endif bx r0 -ENDPROC(cpu_v7_reset) +SYM_FUNC_END(cpu_v7_reset) .popsection /* @@ -72,13 +75,13 @@ ENDPROC(cpu_v7_reset) * * IRQs are already disabled. */ -ENTRY(cpu_v7_do_idle) +SYM_TYPED_FUNC_START(cpu_v7_do_idle) dsb @ WFI may enter a low-power mode wfi ret lr -ENDPROC(cpu_v7_do_idle) +SYM_FUNC_END(cpu_v7_do_idle) -ENTRY(cpu_v7_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_v7_dcache_clean_area) ALT_SMP(W(nop)) @ MP extensions imply L1 PTW ALT_UP_B(1f) ret lr @@ -89,38 +92,39 @@ ENTRY(cpu_v7_dcache_clean_area) bhi 2b dsb ishst ret lr -ENDPROC(cpu_v7_dcache_clean_area) +SYM_FUNC_END(cpu_v7_dcache_clean_area) -#ifdef CONFIG_ARM_PSCI +#if defined(CONFIG_ARM_PSCI) && defined(CONFIG_HARDEN_BRANCH_PREDICTOR) .arch_extension sec -ENTRY(cpu_v7_smc_switch_mm) +SYM_TYPED_FUNC_START(cpu_v7_smc_switch_mm) stmfd sp!, {r0 - r3} movw r0, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1 movt r0, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1 smc #0 ldmfd sp!, {r0 - r3} b cpu_v7_switch_mm -ENDPROC(cpu_v7_smc_switch_mm) +SYM_FUNC_END(cpu_v7_smc_switch_mm) .arch_extension virt -ENTRY(cpu_v7_hvc_switch_mm) +SYM_TYPED_FUNC_START(cpu_v7_hvc_switch_mm) stmfd sp!, {r0 - r3} movw r0, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1 movt r0, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1 hvc #0 ldmfd sp!, {r0 - r3} b cpu_v7_switch_mm -ENDPROC(cpu_v7_hvc_switch_mm) +SYM_FUNC_END(cpu_v7_hvc_switch_mm) #endif -ENTRY(cpu_v7_iciallu_switch_mm) + +SYM_TYPED_FUNC_START(cpu_v7_iciallu_switch_mm) mov r3, #0 mcr p15, 0, r3, c7, c5, 0 @ ICIALLU b cpu_v7_switch_mm -ENDPROC(cpu_v7_iciallu_switch_mm) -ENTRY(cpu_v7_bpiall_switch_mm) +SYM_FUNC_END(cpu_v7_iciallu_switch_mm) +SYM_TYPED_FUNC_START(cpu_v7_bpiall_switch_mm) mov r3, #0 mcr p15, 0, r3, c7, c5, 6 @ flush BTAC/BTB b cpu_v7_switch_mm -ENDPROC(cpu_v7_bpiall_switch_mm) +SYM_FUNC_END(cpu_v7_bpiall_switch_mm) string cpu_v7_name, "ARMv7 Processor" .align @@ -129,7 +133,7 @@ ENDPROC(cpu_v7_bpiall_switch_mm) .globl cpu_v7_suspend_size .equ cpu_v7_suspend_size, 4 * 9 #ifdef CONFIG_ARM_CPU_SUSPEND -ENTRY(cpu_v7_do_suspend) +SYM_TYPED_FUNC_START(cpu_v7_do_suspend) stmfd sp!, {r4 - r11, lr} mrc p15, 0, r4, c13, c0, 0 @ FCSE/PID mrc p15, 0, r5, c13, c0, 3 @ User r/o thread ID @@ -148,9 +152,9 @@ ENTRY(cpu_v7_do_suspend) mrc p15, 0, r10, c1, c0, 2 @ Co-processor access control stmia r0, {r5 - r11} ldmfd sp!, {r4 - r11, pc} -ENDPROC(cpu_v7_do_suspend) +SYM_FUNC_END(cpu_v7_do_suspend) -ENTRY(cpu_v7_do_resume) +SYM_TYPED_FUNC_START(cpu_v7_do_resume) mov ip, #0 mcr p15, 0, ip, c7, c5, 0 @ invalidate I cache mcr p15, 0, ip, c13, c0, 1 @ set reserved context ID @@ -184,22 +188,22 @@ ENTRY(cpu_v7_do_resume) dsb mov r0, r8 @ control register b cpu_resume_mmu -ENDPROC(cpu_v7_do_resume) +SYM_FUNC_END(cpu_v7_do_resume) #endif .globl cpu_ca9mp_suspend_size .equ cpu_ca9mp_suspend_size, cpu_v7_suspend_size + 4 * 2 #ifdef CONFIG_ARM_CPU_SUSPEND -ENTRY(cpu_ca9mp_do_suspend) +SYM_TYPED_FUNC_START(cpu_ca9mp_do_suspend) stmfd sp!, {r4 - r5} mrc p15, 0, r4, c15, c0, 1 @ Diagnostic register mrc p15, 0, r5, c15, c0, 0 @ Power register stmia r0!, {r4 - r5} ldmfd sp!, {r4 - r5} b cpu_v7_do_suspend -ENDPROC(cpu_ca9mp_do_suspend) +SYM_FUNC_END(cpu_ca9mp_do_suspend) -ENTRY(cpu_ca9mp_do_resume) +SYM_TYPED_FUNC_START(cpu_ca9mp_do_resume) ldmia r0!, {r4 - r5} mrc p15, 0, r10, c15, c0, 1 @ Read Diagnostic register teq r4, r10 @ Already restored? @@ -208,7 +212,7 @@ ENTRY(cpu_ca9mp_do_resume) teq r5, r10 @ Already restored? mcrne p15, 0, r5, c15, c0, 0 @ No, so restore it b cpu_v7_do_resume -ENDPROC(cpu_ca9mp_do_resume) +SYM_FUNC_END(cpu_ca9mp_do_resume) #endif #ifdef CONFIG_CPU_PJ4B @@ -218,18 +222,18 @@ ENDPROC(cpu_ca9mp_do_resume) globl_equ cpu_pj4b_proc_fin, cpu_v7_proc_fin globl_equ cpu_pj4b_reset, cpu_v7_reset #ifdef CONFIG_PJ4B_ERRATA_4742 -ENTRY(cpu_pj4b_do_idle) +SYM_TYPED_FUNC_START(cpu_pj4b_do_idle) dsb @ WFI may enter a low-power mode wfi dsb @barrier ret lr -ENDPROC(cpu_pj4b_do_idle) +SYM_FUNC_END(cpu_pj4b_do_idle) #else globl_equ cpu_pj4b_do_idle, cpu_v7_do_idle #endif globl_equ cpu_pj4b_dcache_clean_area, cpu_v7_dcache_clean_area #ifdef CONFIG_ARM_CPU_SUSPEND -ENTRY(cpu_pj4b_do_suspend) +SYM_TYPED_FUNC_START(cpu_pj4b_do_suspend) stmfd sp!, {r6 - r10} mrc p15, 1, r6, c15, c1, 0 @ save CP15 - extra features mrc p15, 1, r7, c15, c2, 0 @ save CP15 - Aux Func Modes Ctrl 0 @@ -239,9 +243,9 @@ ENTRY(cpu_pj4b_do_suspend) stmia r0!, {r6 - r10} ldmfd sp!, {r6 - r10} b cpu_v7_do_suspend -ENDPROC(cpu_pj4b_do_suspend) +SYM_FUNC_END(cpu_pj4b_do_suspend) -ENTRY(cpu_pj4b_do_resume) +SYM_TYPED_FUNC_START(cpu_pj4b_do_resume) ldmia r0!, {r6 - r10} mcr p15, 1, r6, c15, c1, 0 @ restore CP15 - extra features mcr p15, 1, r7, c15, c2, 0 @ restore CP15 - Aux Func Modes Ctrl 0 @@ -249,13 +253,27 @@ ENTRY(cpu_pj4b_do_resume) mcr p15, 1, r9, c15, c1, 1 @ restore CP15 - Aux Debug Modes Ctrl 1 mcr p15, 0, r10, c9, c14, 0 @ restore CP15 - PMC b cpu_v7_do_resume -ENDPROC(cpu_pj4b_do_resume) +SYM_FUNC_END(cpu_pj4b_do_resume) #endif .globl cpu_pj4b_suspend_size .equ cpu_pj4b_suspend_size, cpu_v7_suspend_size + 4 * 5 #endif + @ + @ Invoke the v7_invalidate_l1() function, which adheres to the AAPCS + @ rules, and so it may corrupt registers that we need to preserve. + @ + .macro do_invalidate_l1 + mov r6, r1 + mov r7, r2 + mov r10, lr + bl v7_invalidate_l1 @ corrupts {r0-r3, ip, lr} + mov r1, r6 + mov r2, r7 + mov lr, r10 + .endm + /* * __v7_setup * @@ -277,6 +295,7 @@ __v7_ca5mp_setup: __v7_ca9mp_setup: __v7_cr7mp_setup: __v7_cr8mp_setup: + do_invalidate_l1 mov r10, #(1 << 0) @ Cache/TLB ops broadcasting b 1f __v7_ca7mp_setup: @@ -284,13 +303,9 @@ __v7_ca12mp_setup: __v7_ca15mp_setup: __v7_b15mp_setup: __v7_ca17mp_setup: + do_invalidate_l1 mov r10, #0 -1: adr r0, __v7_setup_stack_ptr - ldr r12, [r0] - add r12, r12, r0 @ the local stack - stmia r12, {r1-r6, lr} @ v7_invalidate_l1 touches r0-r6 - bl v7_invalidate_l1 - ldmia r12, {r1-r6, lr} +1: #ifdef CONFIG_SMP orr r10, r10, #(1 << 6) @ Enable SMP/nAMP mode ALT_SMP(mrc p15, 0, r0, c1, c0, 1) @@ -471,12 +486,7 @@ __v7_pj4b_setup: #endif /* CONFIG_CPU_PJ4B */ __v7_setup: - adr r0, __v7_setup_stack_ptr - ldr r12, [r0] - add r12, r12, r0 @ the local stack - stmia r12, {r1-r6, lr} @ v7_invalidate_l1 touches r0-r6 - bl v7_invalidate_l1 - ldmia r12, {r1-r6, lr} + do_invalidate_l1 __v7_setup_cont: and r0, r9, #0xff000000 @ ARM? @@ -548,17 +558,8 @@ __errata_finish: orr r0, r0, r6 @ set them THUMB( orr r0, r0, #1 << 30 ) @ Thumb exceptions ret lr @ return to head.S:__ret - - .align 2 -__v7_setup_stack_ptr: - .word PHYS_RELATIVE(__v7_setup_stack, .) ENDPROC(__v7_setup) - .bss - .align 2 -__v7_setup_stack: - .space 4 * 7 @ 7 registers - __INITDATA .weak cpu_v7_bugs_init diff --git a/arch/arm/mm/proc-v7m.S b/arch/arm/mm/proc-v7m.S index 84459c1d31b8..d4675603593b 100644 --- a/arch/arm/mm/proc-v7m.S +++ b/arch/arm/mm/proc-v7m.S @@ -8,18 +8,19 @@ * This is the "shell" of the ARMv7-M processor support. */ #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/v7m.h> #include "proc-macros.S" -ENTRY(cpu_v7m_proc_init) +SYM_TYPED_FUNC_START(cpu_v7m_proc_init) ret lr -ENDPROC(cpu_v7m_proc_init) +SYM_FUNC_END(cpu_v7m_proc_init) -ENTRY(cpu_v7m_proc_fin) +SYM_TYPED_FUNC_START(cpu_v7m_proc_fin) ret lr -ENDPROC(cpu_v7m_proc_fin) +SYM_FUNC_END(cpu_v7m_proc_fin) /* * cpu_v7m_reset(loc) @@ -31,9 +32,9 @@ ENDPROC(cpu_v7m_proc_fin) * - loc - location to jump to for soft reset */ .align 5 -ENTRY(cpu_v7m_reset) +SYM_TYPED_FUNC_START(cpu_v7m_reset) ret r0 -ENDPROC(cpu_v7m_reset) +SYM_FUNC_END(cpu_v7m_reset) /* * cpu_v7m_do_idle() @@ -42,36 +43,36 @@ ENDPROC(cpu_v7m_reset) * * IRQs are already disabled. */ -ENTRY(cpu_v7m_do_idle) +SYM_TYPED_FUNC_START(cpu_v7m_do_idle) wfi ret lr -ENDPROC(cpu_v7m_do_idle) +SYM_FUNC_END(cpu_v7m_do_idle) -ENTRY(cpu_v7m_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_v7m_dcache_clean_area) ret lr -ENDPROC(cpu_v7m_dcache_clean_area) +SYM_FUNC_END(cpu_v7m_dcache_clean_area) /* * There is no MMU, so here is nothing to do. */ -ENTRY(cpu_v7m_switch_mm) +SYM_TYPED_FUNC_START(cpu_v7m_switch_mm) ret lr -ENDPROC(cpu_v7m_switch_mm) +SYM_FUNC_END(cpu_v7m_switch_mm) .globl cpu_v7m_suspend_size .equ cpu_v7m_suspend_size, 0 #ifdef CONFIG_ARM_CPU_SUSPEND -ENTRY(cpu_v7m_do_suspend) +SYM_TYPED_FUNC_START(cpu_v7m_do_suspend) ret lr -ENDPROC(cpu_v7m_do_suspend) +SYM_FUNC_END(cpu_v7m_do_suspend) -ENTRY(cpu_v7m_do_resume) +SYM_TYPED_FUNC_START(cpu_v7m_do_resume) ret lr -ENDPROC(cpu_v7m_do_resume) +SYM_FUNC_END(cpu_v7m_do_resume) #endif -ENTRY(cpu_cm7_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_cm7_dcache_clean_area) dcache_line_size r2, r3 movw r3, #:lower16:BASEADDR_V7M_SCB + V7M_SCB_DCCMVAC movt r3, #:upper16:BASEADDR_V7M_SCB + V7M_SCB_DCCMVAC @@ -82,16 +83,16 @@ ENTRY(cpu_cm7_dcache_clean_area) bhi 1b dsb ret lr -ENDPROC(cpu_cm7_dcache_clean_area) +SYM_FUNC_END(cpu_cm7_dcache_clean_area) -ENTRY(cpu_cm7_proc_fin) +SYM_TYPED_FUNC_START(cpu_cm7_proc_fin) movw r2, #:lower16:(BASEADDR_V7M_SCB + V7M_SCB_CCR) movt r2, #:upper16:(BASEADDR_V7M_SCB + V7M_SCB_CCR) ldr r0, [r2] bic r0, r0, #(V7M_SCB_CCR_DC | V7M_SCB_CCR_IC) str r0, [r2] ret lr -ENDPROC(cpu_cm7_proc_fin) +SYM_FUNC_END(cpu_cm7_proc_fin) .section ".init.text", "ax" @@ -194,6 +195,26 @@ ENDPROC(__v7m_setup) .endm /* + * Match ARM Cortex-M55 processor. + */ + .type __v7m_cm55_proc_info, #object +__v7m_cm55_proc_info: + .long 0x410fd220 /* ARM Cortex-M55 0xD22 */ + .long 0xff0ffff0 /* Mask off revision, patch release */ + __v7m_proc __v7m_cm55_proc_info, __v7m_cm7_setup, hwcaps = HWCAP_EDSP, cache_fns = v7m_cache_fns, proc_fns = cm7_processor_functions + .size __v7m_cm55_proc_info, . - __v7m_cm55_proc_info + + /* + * Match ARM Cortex-M33 processor. + */ + .type __v7m_cm33_proc_info, #object +__v7m_cm33_proc_info: + .long 0x410fd210 /* ARM Cortex-M33 0xD21 */ + .long 0xff0ffff0 /* Mask off revision, patch release */ + __v7m_proc __v7m_cm33_proc_info, __v7m_setup, hwcaps = HWCAP_EDSP + .size __v7m_cm33_proc_info, . - __v7m_cm33_proc_info + + /* * Match ARM Cortex-M7 processor. */ .type __v7m_cm7_proc_info, #object diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S index 42eaecc43cfe..14927b380452 100644 --- a/arch/arm/mm/proc-xsc3.S +++ b/arch/arm/mm/proc-xsc3.S @@ -23,9 +23,10 @@ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/hwcap.h> -#include <asm/pgtable.h> #include <asm/pgtable-hwdef.h> #include <asm/page.h> #include <asm/ptrace.h> @@ -79,18 +80,20 @@ * * Nothing too exciting at the moment */ -ENTRY(cpu_xsc3_proc_init) +SYM_TYPED_FUNC_START(cpu_xsc3_proc_init) ret lr +SYM_FUNC_END(cpu_xsc3_proc_init) /* * cpu_xsc3_proc_fin() */ -ENTRY(cpu_xsc3_proc_fin) +SYM_TYPED_FUNC_START(cpu_xsc3_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1800 @ ...IZ........... bic r0, r0, #0x0006 @ .............CA. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_xsc3_proc_fin) /* * cpu_xsc3_reset(loc) @@ -103,7 +106,7 @@ ENTRY(cpu_xsc3_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_xsc3_reset) +SYM_TYPED_FUNC_START(cpu_xsc3_reset) mov r1, #PSR_F_BIT|PSR_I_BIT|SVC_MODE msr cpsr_c, r1 @ reset CPSR mrc p15, 0, r1, c1, c0, 0 @ ctrl register @@ -117,7 +120,7 @@ ENTRY(cpu_xsc3_reset) @ already containing those two last instructions to survive. mcr p15, 0, ip, c8, c7, 0 @ invalidate I and D TLBs ret r0 -ENDPROC(cpu_xsc3_reset) +SYM_FUNC_END(cpu_xsc3_reset) .popsection /* @@ -132,10 +135,11 @@ ENDPROC(cpu_xsc3_reset) */ .align 5 -ENTRY(cpu_xsc3_do_idle) +SYM_TYPED_FUNC_START(cpu_xsc3_do_idle) mov r0, #1 mcr p14, 0, r0, c7, c0, 0 @ go to idle ret lr +SYM_FUNC_END(cpu_xsc3_do_idle) /* ================================= CACHE ================================ */ @@ -144,11 +148,11 @@ ENTRY(cpu_xsc3_do_idle) * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(xsc3_flush_icache_all) +SYM_TYPED_FUNC_START(xsc3_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(xsc3_flush_icache_all) +SYM_FUNC_END(xsc3_flush_icache_all) /* * flush_user_cache_all() @@ -156,15 +160,14 @@ ENDPROC(xsc3_flush_icache_all) * Invalidate all cache entries in a particular address * space. */ -ENTRY(xsc3_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(xsc3_flush_user_cache_all, xsc3_flush_kern_cache_all) /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(xsc3_flush_kern_cache_all) +SYM_TYPED_FUNC_START(xsc3_flush_kern_cache_all) mov r2, #VM_EXEC mov ip, #0 __flush_whole_cache: @@ -174,6 +177,7 @@ __flush_whole_cache: mcrne p15, 0, ip, c7, c10, 4 @ data write barrier mcrne p15, 0, ip, c7, c5, 4 @ prefetch flush ret lr +SYM_FUNC_END(xsc3_flush_kern_cache_all) /* * flush_user_cache_range(start, end, vm_flags) @@ -186,7 +190,7 @@ __flush_whole_cache: * - vma - vma_area_struct describing address space */ .align 5 -ENTRY(xsc3_flush_user_cache_range) +SYM_TYPED_FUNC_START(xsc3_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #MAX_AREA_SIZE @@ -203,6 +207,7 @@ ENTRY(xsc3_flush_user_cache_range) mcrne p15, 0, ip, c7, c10, 4 @ data write barrier mcrne p15, 0, ip, c7, c5, 4 @ prefetch flush ret lr +SYM_FUNC_END(xsc3_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -217,9 +222,13 @@ ENTRY(xsc3_flush_user_cache_range) * Note: single I-cache line invalidation isn't used here since * it also trashes the mini I-cache used by JTAG debuggers. */ -ENTRY(xsc3_coherent_kern_range) -/* FALLTHROUGH */ -ENTRY(xsc3_coherent_user_range) +SYM_TYPED_FUNC_START(xsc3_coherent_kern_range) +#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ + b xsc3_coherent_user_range +#endif +SYM_FUNC_END(xsc3_coherent_kern_range) + +SYM_TYPED_FUNC_START(xsc3_coherent_user_range) bic r0, r0, #CACHELINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean L1 D line add r0, r0, #CACHELINESIZE @@ -230,6 +239,7 @@ ENTRY(xsc3_coherent_user_range) mcr p15, 0, r0, c7, c10, 4 @ data write barrier mcr p15, 0, r0, c7, c5, 4 @ prefetch flush ret lr +SYM_FUNC_END(xsc3_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -240,7 +250,7 @@ ENTRY(xsc3_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(xsc3_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(xsc3_flush_kern_dcache_area) add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean/invalidate L1 D line add r0, r0, #CACHELINESIZE @@ -251,6 +261,7 @@ ENTRY(xsc3_flush_kern_dcache_area) mcr p15, 0, r0, c7, c10, 4 @ data write barrier mcr p15, 0, r0, c7, c5, 4 @ prefetch flush ret lr +SYM_FUNC_END(xsc3_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -301,7 +312,7 @@ xsc3_dma_clean_range: * - start - virtual start address * - end - virtual end address */ -ENTRY(xsc3_dma_flush_range) +SYM_TYPED_FUNC_START(xsc3_dma_flush_range) bic r0, r0, #CACHELINESIZE - 1 1: mcr p15, 0, r0, c7, c14, 1 @ clean/invalidate L1 D line add r0, r0, #CACHELINESIZE @@ -309,6 +320,7 @@ ENTRY(xsc3_dma_flush_range) blo 1b mcr p15, 0, r0, c7, c10, 4 @ data write barrier ret lr +SYM_FUNC_END(xsc3_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -316,13 +328,13 @@ ENTRY(xsc3_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(xsc3_dma_map_area) +SYM_TYPED_FUNC_START(xsc3_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq xsc3_dma_clean_range bcs xsc3_dma_inv_range b xsc3_dma_flush_range -ENDPROC(xsc3_dma_map_area) +SYM_FUNC_END(xsc3_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -330,22 +342,17 @@ ENDPROC(xsc3_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(xsc3_dma_unmap_area) +SYM_TYPED_FUNC_START(xsc3_dma_unmap_area) ret lr -ENDPROC(xsc3_dma_unmap_area) - - .globl xsc3_flush_kern_cache_louis - .equ xsc3_flush_kern_cache_louis, xsc3_flush_kern_cache_all - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions xsc3 +SYM_FUNC_END(xsc3_dma_unmap_area) -ENTRY(cpu_xsc3_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_xsc3_dcache_clean_area) 1: mcr p15, 0, r0, c7, c10, 1 @ clean L1 D line add r0, r0, #CACHELINESIZE subs r1, r1, #CACHELINESIZE bhi 1b ret lr +SYM_FUNC_END(cpu_xsc3_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -357,7 +364,7 @@ ENTRY(cpu_xsc3_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_xsc3_switch_mm) +SYM_TYPED_FUNC_START(cpu_xsc3_switch_mm) clean_d_cache r1, r2 mcr p15, 0, ip, c7, c5, 0 @ invalidate L1 I cache and BTB mcr p15, 0, ip, c7, c10, 4 @ data write barrier @@ -366,6 +373,7 @@ ENTRY(cpu_xsc3_switch_mm) mcr p15, 0, r0, c2, c0, 0 @ load page table pointer mcr p15, 0, ip, c8, c7, 0 @ invalidate I and D TLBs cpwait_ret lr, ip +SYM_FUNC_END(cpu_xsc3_switch_mm) /* * cpu_xsc3_set_pte_ext(ptep, pte, ext) @@ -391,7 +399,7 @@ cpu_xsc3_mt_table: .long 0x00 @ unused .align 5 -ENTRY(cpu_xsc3_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_xsc3_set_pte_ext) xscale_set_pte_ext_prologue tst r1, #L_PTE_SHARED @ shared? @@ -404,6 +412,7 @@ ENTRY(cpu_xsc3_set_pte_ext) xscale_set_pte_ext_epilogue ret lr +SYM_FUNC_END(cpu_xsc3_set_pte_ext) .ltorg .align @@ -411,7 +420,7 @@ ENTRY(cpu_xsc3_set_pte_ext) .globl cpu_xsc3_suspend_size .equ cpu_xsc3_suspend_size, 4 * 6 #ifdef CONFIG_ARM_CPU_SUSPEND -ENTRY(cpu_xsc3_do_suspend) +SYM_TYPED_FUNC_START(cpu_xsc3_do_suspend) stmfd sp!, {r4 - r9, lr} mrc p14, 0, r4, c6, c0, 0 @ clock configuration, for turbo mode mrc p15, 0, r5, c15, c1, 0 @ CP access reg @@ -422,9 +431,9 @@ ENTRY(cpu_xsc3_do_suspend) bic r4, r4, #2 @ clear frequency change bit stmia r0, {r4 - r9} @ store cp regs ldmia sp!, {r4 - r9, pc} -ENDPROC(cpu_xsc3_do_suspend) +SYM_FUNC_END(cpu_xsc3_do_suspend) -ENTRY(cpu_xsc3_do_resume) +SYM_TYPED_FUNC_START(cpu_xsc3_do_resume) ldmia r0, {r4 - r9} @ load cp regs mov ip, #0 mcr p15, 0, ip, c7, c7, 0 @ invalidate I & D caches, BTB @@ -440,7 +449,7 @@ ENTRY(cpu_xsc3_do_resume) mcr p15, 0, r8, c1, c0, 1 @ auxiliary control reg mov r0, r9 @ control register b cpu_resume_mmu -ENDPROC(cpu_xsc3_do_resume) +SYM_FUNC_END(cpu_xsc3_do_resume) #endif .type __xsc3_setup, #function diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S index 18ac5a1f8922..d8462df8020b 100644 --- a/arch/arm/mm/proc-xscale.S +++ b/arch/arm/mm/proc-xscale.S @@ -19,9 +19,10 @@ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> +#include <linux/pgtable.h> #include <asm/assembler.h> #include <asm/hwcap.h> -#include <asm/pgtable.h> #include <asm/pgtable-hwdef.h> #include <asm/page.h> #include <asm/ptrace.h> @@ -111,22 +112,24 @@ clean_addr: .word CLEAN_ADDR * * Nothing too exciting at the moment */ -ENTRY(cpu_xscale_proc_init) +SYM_TYPED_FUNC_START(cpu_xscale_proc_init) @ enable write buffer coalescing. Some bootloader disable it mrc p15, 0, r1, c1, c0, 1 bic r1, r1, #1 mcr p15, 0, r1, c1, c0, 1 ret lr +SYM_FUNC_END(cpu_xscale_proc_init) /* * cpu_xscale_proc_fin() */ -ENTRY(cpu_xscale_proc_fin) +SYM_TYPED_FUNC_START(cpu_xscale_proc_fin) mrc p15, 0, r0, c1, c0, 0 @ ctrl register bic r0, r0, #0x1800 @ ...IZ........... bic r0, r0, #0x0006 @ .............CA. mcr p15, 0, r0, c1, c0, 0 @ disable caches ret lr +SYM_FUNC_END(cpu_xscale_proc_fin) /* * cpu_xscale_reset(loc) @@ -141,7 +144,7 @@ ENTRY(cpu_xscale_proc_fin) */ .align 5 .pushsection .idmap.text, "ax" -ENTRY(cpu_xscale_reset) +SYM_TYPED_FUNC_START(cpu_xscale_reset) mov r1, #PSR_F_BIT|PSR_I_BIT|SVC_MODE msr cpsr_c, r1 @ reset CPSR mcr p15, 0, r1, c10, c4, 1 @ unlock I-TLB @@ -159,7 +162,7 @@ ENTRY(cpu_xscale_reset) @ already containing those two last instructions to survive. mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs ret r0 -ENDPROC(cpu_xscale_reset) +SYM_FUNC_END(cpu_xscale_reset) .popsection /* @@ -174,10 +177,11 @@ ENDPROC(cpu_xscale_reset) */ .align 5 -ENTRY(cpu_xscale_do_idle) +SYM_TYPED_FUNC_START(cpu_xscale_do_idle) mov r0, #1 mcr p14, 0, r0, c7, c0, 0 @ Go to IDLE ret lr +SYM_FUNC_END(cpu_xscale_do_idle) /* ================================= CACHE ================================ */ @@ -186,11 +190,11 @@ ENTRY(cpu_xscale_do_idle) * * Unconditionally clean and invalidate the entire icache. */ -ENTRY(xscale_flush_icache_all) +SYM_TYPED_FUNC_START(xscale_flush_icache_all) mov r0, #0 mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache ret lr -ENDPROC(xscale_flush_icache_all) +SYM_FUNC_END(xscale_flush_icache_all) /* * flush_user_cache_all() @@ -198,15 +202,14 @@ ENDPROC(xscale_flush_icache_all) * Invalidate all cache entries in a particular address * space. */ -ENTRY(xscale_flush_user_cache_all) - /* FALLTHROUGH */ +SYM_FUNC_ALIAS(xscale_flush_user_cache_all, xscale_flush_kern_cache_all) /* * flush_kern_cache_all() * * Clean and invalidate the entire cache. */ -ENTRY(xscale_flush_kern_cache_all) +SYM_TYPED_FUNC_START(xscale_flush_kern_cache_all) mov r2, #VM_EXEC mov ip, #0 __flush_whole_cache: @@ -215,6 +218,7 @@ __flush_whole_cache: mcrne p15, 0, ip, c7, c5, 0 @ Invalidate I cache & BTB mcrne p15, 0, ip, c7, c10, 4 @ Drain Write (& Fill) Buffer ret lr +SYM_FUNC_END(xscale_flush_kern_cache_all) /* * flush_user_cache_range(start, end, vm_flags) @@ -227,7 +231,7 @@ __flush_whole_cache: * - vma - vma_area_struct describing address space */ .align 5 -ENTRY(xscale_flush_user_cache_range) +SYM_TYPED_FUNC_START(xscale_flush_user_cache_range) mov ip, #0 sub r3, r1, r0 @ calculate total size cmp r3, #MAX_AREA_SIZE @@ -244,6 +248,7 @@ ENTRY(xscale_flush_user_cache_range) mcrne p15, 0, ip, c7, c5, 6 @ Invalidate BTB mcrne p15, 0, ip, c7, c10, 4 @ Drain Write (& Fill) Buffer ret lr +SYM_FUNC_END(xscale_flush_user_cache_range) /* * coherent_kern_range(start, end) @@ -258,7 +263,7 @@ ENTRY(xscale_flush_user_cache_range) * Note: single I-cache line invalidation isn't used here since * it also trashes the mini I-cache used by JTAG debuggers. */ -ENTRY(xscale_coherent_kern_range) +SYM_TYPED_FUNC_START(xscale_coherent_kern_range) bic r0, r0, #CACHELINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHELINESIZE @@ -268,6 +273,7 @@ ENTRY(xscale_coherent_kern_range) mcr p15, 0, r0, c7, c5, 0 @ Invalidate I cache & BTB mcr p15, 0, r0, c7, c10, 4 @ Drain Write (& Fill) Buffer ret lr +SYM_FUNC_END(xscale_coherent_kern_range) /* * coherent_user_range(start, end) @@ -279,7 +285,7 @@ ENTRY(xscale_coherent_kern_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(xscale_coherent_user_range) +SYM_TYPED_FUNC_START(xscale_coherent_user_range) bic r0, r0, #CACHELINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry mcr p15, 0, r0, c7, c5, 1 @ Invalidate I cache entry @@ -290,6 +296,7 @@ ENTRY(xscale_coherent_user_range) mcr p15, 0, r0, c7, c5, 6 @ Invalidate BTB mcr p15, 0, r0, c7, c10, 4 @ Drain Write (& Fill) Buffer ret lr +SYM_FUNC_END(xscale_coherent_user_range) /* * flush_kern_dcache_area(void *addr, size_t size) @@ -300,7 +307,7 @@ ENTRY(xscale_coherent_user_range) * - addr - kernel address * - size - region size */ -ENTRY(xscale_flush_kern_dcache_area) +SYM_TYPED_FUNC_START(xscale_flush_kern_dcache_area) add r1, r0, r1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry mcr p15, 0, r0, c7, c6, 1 @ invalidate D entry @@ -311,6 +318,7 @@ ENTRY(xscale_flush_kern_dcache_area) mcr p15, 0, r0, c7, c5, 0 @ Invalidate I cache & BTB mcr p15, 0, r0, c7, c10, 4 @ Drain Write (& Fill) Buffer ret lr +SYM_FUNC_END(xscale_flush_kern_dcache_area) /* * dma_inv_range(start, end) @@ -361,7 +369,7 @@ xscale_dma_clean_range: * - start - virtual start address * - end - virtual end address */ -ENTRY(xscale_dma_flush_range) +SYM_TYPED_FUNC_START(xscale_dma_flush_range) bic r0, r0, #CACHELINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry mcr p15, 0, r0, c7, c6, 1 @ invalidate D entry @@ -370,6 +378,7 @@ ENTRY(xscale_dma_flush_range) blo 1b mcr p15, 0, r0, c7, c10, 4 @ Drain Write (& Fill) Buffer ret lr +SYM_FUNC_END(xscale_dma_flush_range) /* * dma_map_area(start, size, dir) @@ -377,13 +386,27 @@ ENTRY(xscale_dma_flush_range) * - size - size of region * - dir - DMA direction */ -ENTRY(xscale_dma_map_area) +SYM_TYPED_FUNC_START(xscale_dma_map_area) add r1, r1, r0 cmp r2, #DMA_TO_DEVICE beq xscale_dma_clean_range bcs xscale_dma_inv_range b xscale_dma_flush_range -ENDPROC(xscale_dma_map_area) +SYM_FUNC_END(xscale_dma_map_area) + +/* + * On stepping A0/A1 of the 80200, invalidating D-cache by line doesn't + * clear the dirty bits, which means that if we invalidate a dirty line, + * the dirty data can still be written back to external memory later on. + * + * The recommended workaround is to always do a clean D-cache line before + * doing an invalidate D-cache line, so on the affected processors, + * dma_inv_range() is implemented as dma_flush_range(). + * + * See erratum #25 of "Intel 80200 Processor Specification Update", + * revision January 22, 2003, available at: + * http://www.intel.com/design/iio/specupdt/273415.htm + */ /* * dma_map_area(start, size, dir) @@ -391,12 +414,12 @@ ENDPROC(xscale_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(xscale_80200_A0_A1_dma_map_area) +SYM_TYPED_FUNC_START(xscale_80200_A0_A1_dma_map_area) add r1, r1, r0 teq r2, #DMA_TO_DEVICE beq xscale_dma_clean_range b xscale_dma_flush_range -ENDPROC(xscale_80200_A0_A1_dma_map_area) +SYM_FUNC_END(xscale_80200_A0_A1_dma_map_area) /* * dma_unmap_area(start, size, dir) @@ -404,59 +427,17 @@ ENDPROC(xscale_80200_A0_A1_dma_map_area) * - size - size of region * - dir - DMA direction */ -ENTRY(xscale_dma_unmap_area) +SYM_TYPED_FUNC_START(xscale_dma_unmap_area) ret lr -ENDPROC(xscale_dma_unmap_area) - - .globl xscale_flush_kern_cache_louis - .equ xscale_flush_kern_cache_louis, xscale_flush_kern_cache_all +SYM_FUNC_END(xscale_dma_unmap_area) - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions xscale - -/* - * On stepping A0/A1 of the 80200, invalidating D-cache by line doesn't - * clear the dirty bits, which means that if we invalidate a dirty line, - * the dirty data can still be written back to external memory later on. - * - * The recommended workaround is to always do a clean D-cache line before - * doing an invalidate D-cache line, so on the affected processors, - * dma_inv_range() is implemented as dma_flush_range(). - * - * See erratum #25 of "Intel 80200 Processor Specification Update", - * revision January 22, 2003, available at: - * http://www.intel.com/design/iio/specupdt/273415.htm - */ -.macro a0_alias basename - .globl xscale_80200_A0_A1_\basename - .type xscale_80200_A0_A1_\basename , %function - .equ xscale_80200_A0_A1_\basename , xscale_\basename -.endm - -/* - * Most of the cache functions are unchanged for these processor revisions. - * Export suitable alias symbols for the unchanged functions: - */ - a0_alias flush_icache_all - a0_alias flush_user_cache_all - a0_alias flush_kern_cache_all - a0_alias flush_kern_cache_louis - a0_alias flush_user_cache_range - a0_alias coherent_kern_range - a0_alias coherent_user_range - a0_alias flush_kern_dcache_area - a0_alias dma_flush_range - a0_alias dma_unmap_area - - @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) - define_cache_functions xscale_80200_A0_A1 - -ENTRY(cpu_xscale_dcache_clean_area) +SYM_TYPED_FUNC_START(cpu_xscale_dcache_clean_area) 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHELINESIZE subs r1, r1, #CACHELINESIZE bhi 1b ret lr +SYM_FUNC_END(cpu_xscale_dcache_clean_area) /* =============================== PageTable ============================== */ @@ -468,13 +449,14 @@ ENTRY(cpu_xscale_dcache_clean_area) * pgd: new page tables */ .align 5 -ENTRY(cpu_xscale_switch_mm) +SYM_TYPED_FUNC_START(cpu_xscale_switch_mm) clean_d_cache r1, r2 mcr p15, 0, ip, c7, c5, 0 @ Invalidate I cache & BTB mcr p15, 0, ip, c7, c10, 4 @ Drain Write (& Fill) Buffer mcr p15, 0, r0, c2, c0, 0 @ load page table pointer mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs cpwait_ret lr, ip +SYM_FUNC_END(cpu_xscale_switch_mm) /* * cpu_xscale_set_pte_ext(ptep, pte, ext) @@ -502,7 +484,7 @@ cpu_xscale_mt_table: .long 0x00 @ unused .align 5 -ENTRY(cpu_xscale_set_pte_ext) +SYM_TYPED_FUNC_START(cpu_xscale_set_pte_ext) xscale_set_pte_ext_prologue @ @@ -520,6 +502,7 @@ ENTRY(cpu_xscale_set_pte_ext) xscale_set_pte_ext_epilogue ret lr +SYM_FUNC_END(cpu_xscale_set_pte_ext) .ltorg .align @@ -527,7 +510,7 @@ ENTRY(cpu_xscale_set_pte_ext) .globl cpu_xscale_suspend_size .equ cpu_xscale_suspend_size, 4 * 6 #ifdef CONFIG_ARM_CPU_SUSPEND -ENTRY(cpu_xscale_do_suspend) +SYM_TYPED_FUNC_START(cpu_xscale_do_suspend) stmfd sp!, {r4 - r9, lr} mrc p14, 0, r4, c6, c0, 0 @ clock configuration, for turbo mode mrc p15, 0, r5, c15, c1, 0 @ CP access reg @@ -538,9 +521,9 @@ ENTRY(cpu_xscale_do_suspend) bic r4, r4, #2 @ clear frequency change bit stmia r0, {r4 - r9} @ store cp regs ldmfd sp!, {r4 - r9, pc} -ENDPROC(cpu_xscale_do_suspend) +SYM_FUNC_END(cpu_xscale_do_suspend) -ENTRY(cpu_xscale_do_resume) +SYM_TYPED_FUNC_START(cpu_xscale_do_resume) ldmia r0, {r4 - r9} @ load cp regs mov ip, #0 mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs @@ -553,7 +536,7 @@ ENTRY(cpu_xscale_do_resume) mcr p15, 0, r8, c1, c0, 1 @ auxiliary control reg mov r0, r9 @ control register b cpu_resume_mmu -ENDPROC(cpu_xscale_do_resume) +SYM_FUNC_END(cpu_xscale_do_resume) #endif .type __xscale_setup, #function diff --git a/arch/arm/mm/proc.c b/arch/arm/mm/proc.c new file mode 100644 index 000000000000..2027845efefb --- /dev/null +++ b/arch/arm/mm/proc.c @@ -0,0 +1,500 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file defines C prototypes for the low-level processor assembly functions + * and creates a reference for CFI. This needs to be done for every assembly + * processor ("proc") function that is called from C but does not have a + * corresponding C implementation. + * + * Processors are listed in the order they appear in the Makefile. + * + * Functions are listed if and only if they see use on the target CPU, and in + * the order they are defined in struct processor. + */ +#include <asm/proc-fns.h> + +#ifdef CONFIG_CPU_ARM7TDMI +void cpu_arm7tdmi_proc_init(void); +__ADDRESSABLE(cpu_arm7tdmi_proc_init); +void cpu_arm7tdmi_proc_fin(void); +__ADDRESSABLE(cpu_arm7tdmi_proc_fin); +void cpu_arm7tdmi_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm7tdmi_reset); +int cpu_arm7tdmi_do_idle(void); +__ADDRESSABLE(cpu_arm7tdmi_do_idle); +void cpu_arm7tdmi_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm7tdmi_dcache_clean_area); +void cpu_arm7tdmi_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm7tdmi_switch_mm); +#endif + +#ifdef CONFIG_CPU_ARM720T +void cpu_arm720_proc_init(void); +__ADDRESSABLE(cpu_arm720_proc_init); +void cpu_arm720_proc_fin(void); +__ADDRESSABLE(cpu_arm720_proc_fin); +void cpu_arm720_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm720_reset); +int cpu_arm720_do_idle(void); +__ADDRESSABLE(cpu_arm720_do_idle); +void cpu_arm720_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm720_dcache_clean_area); +void cpu_arm720_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm720_switch_mm); +void cpu_arm720_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_arm720_set_pte_ext); +#endif + +#ifdef CONFIG_CPU_ARM740T +void cpu_arm740_proc_init(void); +__ADDRESSABLE(cpu_arm740_proc_init); +void cpu_arm740_proc_fin(void); +__ADDRESSABLE(cpu_arm740_proc_fin); +void cpu_arm740_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm740_reset); +int cpu_arm740_do_idle(void); +__ADDRESSABLE(cpu_arm740_do_idle); +void cpu_arm740_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm740_dcache_clean_area); +void cpu_arm740_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm740_switch_mm); +#endif + +#ifdef CONFIG_CPU_ARM9TDMI +void cpu_arm9tdmi_proc_init(void); +__ADDRESSABLE(cpu_arm9tdmi_proc_init); +void cpu_arm9tdmi_proc_fin(void); +__ADDRESSABLE(cpu_arm9tdmi_proc_fin); +void cpu_arm9tdmi_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm9tdmi_reset); +int cpu_arm9tdmi_do_idle(void); +__ADDRESSABLE(cpu_arm9tdmi_do_idle); +void cpu_arm9tdmi_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm9tdmi_dcache_clean_area); +void cpu_arm9tdmi_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm9tdmi_switch_mm); +#endif + +#ifdef CONFIG_CPU_ARM920T +void cpu_arm920_proc_init(void); +__ADDRESSABLE(cpu_arm920_proc_init); +void cpu_arm920_proc_fin(void); +__ADDRESSABLE(cpu_arm920_proc_fin); +void cpu_arm920_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm920_reset); +int cpu_arm920_do_idle(void); +__ADDRESSABLE(cpu_arm920_do_idle); +void cpu_arm920_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm920_dcache_clean_area); +void cpu_arm920_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm920_switch_mm); +void cpu_arm920_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_arm920_set_pte_ext); +#ifdef CONFIG_ARM_CPU_SUSPEND +void cpu_arm920_do_suspend(void *); +__ADDRESSABLE(cpu_arm920_do_suspend); +void cpu_arm920_do_resume(void *); +__ADDRESSABLE(cpu_arm920_do_resume); +#endif /* CONFIG_ARM_CPU_SUSPEND */ +#endif /* CONFIG_CPU_ARM920T */ + +#ifdef CONFIG_CPU_ARM922T +void cpu_arm922_proc_init(void); +__ADDRESSABLE(cpu_arm922_proc_init); +void cpu_arm922_proc_fin(void); +__ADDRESSABLE(cpu_arm922_proc_fin); +void cpu_arm922_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm922_reset); +int cpu_arm922_do_idle(void); +__ADDRESSABLE(cpu_arm922_do_idle); +void cpu_arm922_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm922_dcache_clean_area); +void cpu_arm922_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm922_switch_mm); +void cpu_arm922_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_arm922_set_pte_ext); +#endif + +#ifdef CONFIG_CPU_ARM925T +void cpu_arm925_proc_init(void); +__ADDRESSABLE(cpu_arm925_proc_init); +void cpu_arm925_proc_fin(void); +__ADDRESSABLE(cpu_arm925_proc_fin); +void cpu_arm925_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm925_reset); +int cpu_arm925_do_idle(void); +__ADDRESSABLE(cpu_arm925_do_idle); +void cpu_arm925_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm925_dcache_clean_area); +void cpu_arm925_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm925_switch_mm); +void cpu_arm925_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_arm925_set_pte_ext); +#endif + +#ifdef CONFIG_CPU_ARM926T +void cpu_arm926_proc_init(void); +__ADDRESSABLE(cpu_arm926_proc_init); +void cpu_arm926_proc_fin(void); +__ADDRESSABLE(cpu_arm926_proc_fin); +void cpu_arm926_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm926_reset); +int cpu_arm926_do_idle(void); +__ADDRESSABLE(cpu_arm926_do_idle); +void cpu_arm926_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm926_dcache_clean_area); +void cpu_arm926_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm926_switch_mm); +void cpu_arm926_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_arm926_set_pte_ext); +#ifdef CONFIG_ARM_CPU_SUSPEND +void cpu_arm926_do_suspend(void *); +__ADDRESSABLE(cpu_arm926_do_suspend); +void cpu_arm926_do_resume(void *); +__ADDRESSABLE(cpu_arm926_do_resume); +#endif /* CONFIG_ARM_CPU_SUSPEND */ +#endif /* CONFIG_CPU_ARM926T */ + +#ifdef CONFIG_CPU_ARM940T +void cpu_arm940_proc_init(void); +__ADDRESSABLE(cpu_arm940_proc_init); +void cpu_arm940_proc_fin(void); +__ADDRESSABLE(cpu_arm940_proc_fin); +void cpu_arm940_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm940_reset); +int cpu_arm940_do_idle(void); +__ADDRESSABLE(cpu_arm940_do_idle); +void cpu_arm940_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm940_dcache_clean_area); +void cpu_arm940_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm940_switch_mm); +#endif + +#ifdef CONFIG_CPU_ARM946E +void cpu_arm946_proc_init(void); +__ADDRESSABLE(cpu_arm946_proc_init); +void cpu_arm946_proc_fin(void); +__ADDRESSABLE(cpu_arm946_proc_fin); +void cpu_arm946_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm946_reset); +int cpu_arm946_do_idle(void); +__ADDRESSABLE(cpu_arm946_do_idle); +void cpu_arm946_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm946_dcache_clean_area); +void cpu_arm946_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm946_switch_mm); +#endif + +#ifdef CONFIG_CPU_FA526 +void cpu_fa526_proc_init(void); +__ADDRESSABLE(cpu_fa526_proc_init); +void cpu_fa526_proc_fin(void); +__ADDRESSABLE(cpu_fa526_proc_fin); +void cpu_fa526_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_fa526_reset); +int cpu_fa526_do_idle(void); +__ADDRESSABLE(cpu_fa526_do_idle); +void cpu_fa526_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_fa526_dcache_clean_area); +void cpu_fa526_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_fa526_switch_mm); +void cpu_fa526_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_fa526_set_pte_ext); +#endif + +#ifdef CONFIG_CPU_ARM1020 +void cpu_arm1020_proc_init(void); +__ADDRESSABLE(cpu_arm1020_proc_init); +void cpu_arm1020_proc_fin(void); +__ADDRESSABLE(cpu_arm1020_proc_fin); +void cpu_arm1020_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm1020_reset); +int cpu_arm1020_do_idle(void); +__ADDRESSABLE(cpu_arm1020_do_idle); +void cpu_arm1020_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm1020_dcache_clean_area); +void cpu_arm1020_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm1020_switch_mm); +void cpu_arm1020_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_arm1020_set_pte_ext); +#endif + +#ifdef CONFIG_CPU_ARM1020E +void cpu_arm1020e_proc_init(void); +__ADDRESSABLE(cpu_arm1020e_proc_init); +void cpu_arm1020e_proc_fin(void); +__ADDRESSABLE(cpu_arm1020e_proc_fin); +void cpu_arm1020e_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm1020e_reset); +int cpu_arm1020e_do_idle(void); +__ADDRESSABLE(cpu_arm1020e_do_idle); +void cpu_arm1020e_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm1020e_dcache_clean_area); +void cpu_arm1020e_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm1020e_switch_mm); +void cpu_arm1020e_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_arm1020e_set_pte_ext); +#endif + +#ifdef CONFIG_CPU_ARM1022 +void cpu_arm1022_proc_init(void); +__ADDRESSABLE(cpu_arm1022_proc_init); +void cpu_arm1022_proc_fin(void); +__ADDRESSABLE(cpu_arm1022_proc_fin); +void cpu_arm1022_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm1022_reset); +int cpu_arm1022_do_idle(void); +__ADDRESSABLE(cpu_arm1022_do_idle); +void cpu_arm1022_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm1022_dcache_clean_area); +void cpu_arm1022_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm1022_switch_mm); +void cpu_arm1022_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_arm1022_set_pte_ext); +#endif + +#ifdef CONFIG_CPU_ARM1026 +void cpu_arm1026_proc_init(void); +__ADDRESSABLE(cpu_arm1026_proc_init); +void cpu_arm1026_proc_fin(void); +__ADDRESSABLE(cpu_arm1026_proc_fin); +void cpu_arm1026_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_arm1026_reset); +int cpu_arm1026_do_idle(void); +__ADDRESSABLE(cpu_arm1026_do_idle); +void cpu_arm1026_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_arm1026_dcache_clean_area); +void cpu_arm1026_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_arm1026_switch_mm); +void cpu_arm1026_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_arm1026_set_pte_ext); +#endif + +#ifdef CONFIG_CPU_SA110 +void cpu_sa110_proc_init(void); +__ADDRESSABLE(cpu_sa110_proc_init); +void cpu_sa110_proc_fin(void); +__ADDRESSABLE(cpu_sa110_proc_fin); +void cpu_sa110_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_sa110_reset); +int cpu_sa110_do_idle(void); +__ADDRESSABLE(cpu_sa110_do_idle); +void cpu_sa110_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_sa110_dcache_clean_area); +void cpu_sa110_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_sa110_switch_mm); +void cpu_sa110_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_sa110_set_pte_ext); +#endif + +#ifdef CONFIG_CPU_SA1100 +void cpu_sa1100_proc_init(void); +__ADDRESSABLE(cpu_sa1100_proc_init); +void cpu_sa1100_proc_fin(void); +__ADDRESSABLE(cpu_sa1100_proc_fin); +void cpu_sa1100_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_sa1100_reset); +int cpu_sa1100_do_idle(void); +__ADDRESSABLE(cpu_sa1100_do_idle); +void cpu_sa1100_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_sa1100_dcache_clean_area); +void cpu_sa1100_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_sa1100_switch_mm); +void cpu_sa1100_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_sa1100_set_pte_ext); +#ifdef CONFIG_ARM_CPU_SUSPEND +void cpu_sa1100_do_suspend(void *); +__ADDRESSABLE(cpu_sa1100_do_suspend); +void cpu_sa1100_do_resume(void *); +__ADDRESSABLE(cpu_sa1100_do_resume); +#endif /* CONFIG_ARM_CPU_SUSPEND */ +#endif /* CONFIG_CPU_SA1100 */ + +#ifdef CONFIG_CPU_XSCALE +void cpu_xscale_proc_init(void); +__ADDRESSABLE(cpu_xscale_proc_init); +void cpu_xscale_proc_fin(void); +__ADDRESSABLE(cpu_xscale_proc_fin); +void cpu_xscale_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_xscale_reset); +int cpu_xscale_do_idle(void); +__ADDRESSABLE(cpu_xscale_do_idle); +void cpu_xscale_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_xscale_dcache_clean_area); +void cpu_xscale_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_xscale_switch_mm); +void cpu_xscale_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_xscale_set_pte_ext); +#ifdef CONFIG_ARM_CPU_SUSPEND +void cpu_xscale_do_suspend(void *); +__ADDRESSABLE(cpu_xscale_do_suspend); +void cpu_xscale_do_resume(void *); +__ADDRESSABLE(cpu_xscale_do_resume); +#endif /* CONFIG_ARM_CPU_SUSPEND */ +#endif /* CONFIG_CPU_XSCALE */ + +#ifdef CONFIG_CPU_XSC3 +void cpu_xsc3_proc_init(void); +__ADDRESSABLE(cpu_xsc3_proc_init); +void cpu_xsc3_proc_fin(void); +__ADDRESSABLE(cpu_xsc3_proc_fin); +void cpu_xsc3_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_xsc3_reset); +int cpu_xsc3_do_idle(void); +__ADDRESSABLE(cpu_xsc3_do_idle); +void cpu_xsc3_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_xsc3_dcache_clean_area); +void cpu_xsc3_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_xsc3_switch_mm); +void cpu_xsc3_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_xsc3_set_pte_ext); +#ifdef CONFIG_ARM_CPU_SUSPEND +void cpu_xsc3_do_suspend(void *); +__ADDRESSABLE(cpu_xsc3_do_suspend); +void cpu_xsc3_do_resume(void *); +__ADDRESSABLE(cpu_xsc3_do_resume); +#endif /* CONFIG_ARM_CPU_SUSPEND */ +#endif /* CONFIG_CPU_XSC3 */ + +#ifdef CONFIG_CPU_MOHAWK +void cpu_mohawk_proc_init(void); +__ADDRESSABLE(cpu_mohawk_proc_init); +void cpu_mohawk_proc_fin(void); +__ADDRESSABLE(cpu_mohawk_proc_fin); +void cpu_mohawk_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_mohawk_reset); +int cpu_mohawk_do_idle(void); +__ADDRESSABLE(cpu_mohawk_do_idle); +void cpu_mohawk_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_mohawk_dcache_clean_area); +void cpu_mohawk_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_mohawk_switch_mm); +void cpu_mohawk_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_mohawk_set_pte_ext); +#ifdef CONFIG_ARM_CPU_SUSPEND +void cpu_mohawk_do_suspend(void *); +__ADDRESSABLE(cpu_mohawk_do_suspend); +void cpu_mohawk_do_resume(void *); +__ADDRESSABLE(cpu_mohawk_do_resume); +#endif /* CONFIG_ARM_CPU_SUSPEND */ +#endif /* CONFIG_CPU_MOHAWK */ + +#ifdef CONFIG_CPU_FEROCEON +void cpu_feroceon_proc_init(void); +__ADDRESSABLE(cpu_feroceon_proc_init); +void cpu_feroceon_proc_fin(void); +__ADDRESSABLE(cpu_feroceon_proc_fin); +void cpu_feroceon_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_feroceon_reset); +int cpu_feroceon_do_idle(void); +__ADDRESSABLE(cpu_feroceon_do_idle); +void cpu_feroceon_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_feroceon_dcache_clean_area); +void cpu_feroceon_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_feroceon_switch_mm); +void cpu_feroceon_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_feroceon_set_pte_ext); +#ifdef CONFIG_ARM_CPU_SUSPEND +void cpu_feroceon_do_suspend(void *); +__ADDRESSABLE(cpu_feroceon_do_suspend); +void cpu_feroceon_do_resume(void *); +__ADDRESSABLE(cpu_feroceon_do_resume); +#endif /* CONFIG_ARM_CPU_SUSPEND */ +#endif /* CONFIG_CPU_FEROCEON */ + +#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K) +void cpu_v6_proc_init(void); +__ADDRESSABLE(cpu_v6_proc_init); +void cpu_v6_proc_fin(void); +__ADDRESSABLE(cpu_v6_proc_fin); +void cpu_v6_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_v6_reset); +int cpu_v6_do_idle(void); +__ADDRESSABLE(cpu_v6_do_idle); +void cpu_v6_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_v6_dcache_clean_area); +void cpu_v6_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_v6_switch_mm); +void cpu_v6_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_v6_set_pte_ext); +#ifdef CONFIG_ARM_CPU_SUSPEND +void cpu_v6_do_suspend(void *); +__ADDRESSABLE(cpu_v6_do_suspend); +void cpu_v6_do_resume(void *); +__ADDRESSABLE(cpu_v6_do_resume); +#endif /* CONFIG_ARM_CPU_SUSPEND */ +#endif /* CPU_V6 */ + +#ifdef CONFIG_CPU_V7 +void cpu_v7_proc_init(void); +__ADDRESSABLE(cpu_v7_proc_init); +void cpu_v7_proc_fin(void); +__ADDRESSABLE(cpu_v7_proc_fin); +void cpu_v7_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_v7_reset); +int cpu_v7_do_idle(void); +__ADDRESSABLE(cpu_v7_do_idle); +#ifdef CONFIG_PJ4B_ERRATA_4742 +int cpu_pj4b_do_idle(void); +__ADDRESSABLE(cpu_pj4b_do_idle); +#endif +void cpu_v7_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_v7_dcache_clean_area); +void cpu_v7_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +/* Special switch_mm() callbacks to work around bugs in v7 */ +__ADDRESSABLE(cpu_v7_switch_mm); +void cpu_v7_iciallu_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_v7_iciallu_switch_mm); +void cpu_v7_bpiall_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_v7_bpiall_switch_mm); +#ifdef CONFIG_ARM_LPAE +void cpu_v7_set_pte_ext(pte_t *ptep, pte_t pte); +#else +void cpu_v7_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +#endif +__ADDRESSABLE(cpu_v7_set_pte_ext); +#ifdef CONFIG_ARM_CPU_SUSPEND +void cpu_v7_do_suspend(void *); +__ADDRESSABLE(cpu_v7_do_suspend); +void cpu_v7_do_resume(void *); +__ADDRESSABLE(cpu_v7_do_resume); +/* Special versions of suspend and resume for the CA9MP cores */ +void cpu_ca9mp_do_suspend(void *); +__ADDRESSABLE(cpu_ca9mp_do_suspend); +void cpu_ca9mp_do_resume(void *); +__ADDRESSABLE(cpu_ca9mp_do_resume); +/* Special versions of suspend and resume for the Marvell PJ4B cores */ +#ifdef CONFIG_CPU_PJ4B +void cpu_pj4b_do_suspend(void *); +__ADDRESSABLE(cpu_pj4b_do_suspend); +void cpu_pj4b_do_resume(void *); +__ADDRESSABLE(cpu_pj4b_do_resume); +#endif /* CONFIG_CPU_PJ4B */ +#endif /* CONFIG_ARM_CPU_SUSPEND */ +#endif /* CONFIG_CPU_V7 */ + +#ifdef CONFIG_CPU_V7M +void cpu_v7m_proc_init(void); +__ADDRESSABLE(cpu_v7m_proc_init); +void cpu_v7m_proc_fin(void); +__ADDRESSABLE(cpu_v7m_proc_fin); +void cpu_v7m_reset(unsigned long addr, bool hvc); +__ADDRESSABLE(cpu_v7m_reset); +int cpu_v7m_do_idle(void); +__ADDRESSABLE(cpu_v7m_do_idle); +void cpu_v7m_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_v7m_dcache_clean_area); +void cpu_v7m_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +__ADDRESSABLE(cpu_v7m_switch_mm); +void cpu_v7m_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +__ADDRESSABLE(cpu_v7m_set_pte_ext); +#ifdef CONFIG_ARM_CPU_SUSPEND +void cpu_v7m_do_suspend(void *); +__ADDRESSABLE(cpu_v7m_do_suspend); +void cpu_v7m_do_resume(void *); +__ADDRESSABLE(cpu_v7m_do_resume); +#endif /* CONFIG_ARM_CPU_SUSPEND */ +void cpu_cm7_proc_fin(void); +__ADDRESSABLE(cpu_cm7_proc_fin); +void cpu_cm7_dcache_clean_area(void *addr, int size); +__ADDRESSABLE(cpu_cm7_dcache_clean_area); +#endif /* CONFIG_CPU_V7M */ diff --git a/arch/arm/mm/ptdump_debugfs.c b/arch/arm/mm/ptdump_debugfs.c index 598b636615a2..318de969ae0f 100644 --- a/arch/arm/mm/ptdump_debugfs.c +++ b/arch/arm/mm/ptdump_debugfs.c @@ -11,20 +11,9 @@ static int ptdump_show(struct seq_file *m, void *v) ptdump_walk_pgd(m, info); return 0; } +DEFINE_SHOW_ATTRIBUTE(ptdump); -static int ptdump_open(struct inode *inode, struct file *file) -{ - return single_open(file, ptdump_show, inode->i_private); -} - -static const struct file_operations ptdump_fops = { - .open = ptdump_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -void ptdump_debugfs_register(struct ptdump_info *info, const char *name) +void __init ptdump_debugfs_register(struct ptdump_info *info, const char *name) { debugfs_create_file(name, 0400, NULL, info, &ptdump_fops); } diff --git a/arch/arm/mm/pv-fixup-asm.S b/arch/arm/mm/pv-fixup-asm.S index 769778928356..1d9f52c71ad0 100644 --- a/arch/arm/mm/pv-fixup-asm.S +++ b/arch/arm/mm/pv-fixup-asm.S @@ -6,10 +6,10 @@ * for Keystone 2 */ #include <linux/linkage.h> +#include <linux/pgtable.h> #include <asm/asm-offsets.h> #include <asm/cp15.h> -#include <asm/memory.h> -#include <asm/pgtable.h> +#include <asm/page.h> .section ".idmap.text", "ax" @@ -29,7 +29,7 @@ ENTRY(lpae_pgtables_remap_asm) ldr r6, =(_end - 1) add r7, r2, #0x1000 add r6, r7, r6, lsr #SECTION_SHIFT - L2_ORDER - add r7, r7, #PAGE_OFFSET >> (SECTION_SHIFT - L2_ORDER) + add r7, r7, #KERNEL_OFFSET >> (SECTION_SHIFT - L2_ORDER) 1: ldrd r4, r5, [r7] adds r4, r4, r0 adc r5, r5, r1 @@ -39,8 +39,8 @@ ENTRY(lpae_pgtables_remap_asm) /* Update level 2 entries for the boot data */ add r7, r2, #0x1000 - add r7, r7, r3, lsr #SECTION_SHIFT - L2_ORDER - bic r7, r7, #(1 << L2_ORDER) - 1 + movw r3, #FDT_FIXED_BASE >> (SECTION_SHIFT - L2_ORDER) + add r7, r7, r3 ldrd r4, r5, [r7] adds r4, r4, r0 adc r5, r5, r1 diff --git a/arch/arm/mm/tcm.h b/arch/arm/mm/tcm.h deleted file mode 100644 index 6b80a760d875..000000000000 --- a/arch/arm/mm/tcm.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2008-2009 ST-Ericsson AB - * TCM memory handling for ARM systems - * - * Author: Linus Walleij <linus.walleij@stericsson.com> - * Author: Rickard Andersson <rickard.andersson@stericsson.com> - */ - -#ifdef CONFIG_HAVE_TCM -void __init tcm_init(void); -#else -/* No TCM support, just blank inlines to be optimized out */ -static inline void tcm_init(void) -{ -} -#endif diff --git a/arch/arm/mm/tlb-fa.S b/arch/arm/mm/tlb-fa.S index def6161ec452..85a6fe766b21 100644 --- a/arch/arm/mm/tlb-fa.S +++ b/arch/arm/mm/tlb-fa.S @@ -15,6 +15,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/tlbflush.h> @@ -31,7 +32,7 @@ * - mm - mm_struct describing address space */ .align 4 -ENTRY(fa_flush_user_tlb_range) +SYM_TYPED_FUNC_START(fa_flush_user_tlb_range) vma_vm_mm ip, r2 act_mm r3 @ get current->active_mm eors r3, ip, r3 @ == mm ? @@ -46,9 +47,10 @@ ENTRY(fa_flush_user_tlb_range) blo 1b mcr p15, 0, r3, c7, c10, 4 @ data write barrier ret lr +SYM_FUNC_END(fa_flush_user_tlb_range) -ENTRY(fa_flush_kern_tlb_range) +SYM_TYPED_FUNC_START(fa_flush_kern_tlb_range) mov r3, #0 mcr p15, 0, r3, c7, c10, 4 @ drain WB bic r0, r0, #0x0ff @@ -60,8 +62,4 @@ ENTRY(fa_flush_kern_tlb_range) mcr p15, 0, r3, c7, c10, 4 @ data write barrier mcr p15, 0, r3, c7, c5, 4 @ prefetch flush (isb) ret lr - - __INITDATA - - /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */ - define_tlb_functions fa, fa_tlb_flags +SYM_FUNC_END(fa_flush_kern_tlb_range) diff --git a/arch/arm/mm/tlb-v4.S b/arch/arm/mm/tlb-v4.S index b962b4e75158..09ff69008d94 100644 --- a/arch/arm/mm/tlb-v4.S +++ b/arch/arm/mm/tlb-v4.S @@ -11,6 +11,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/tlbflush.h> @@ -27,7 +28,7 @@ * - mm - mm_struct describing address space */ .align 5 -ENTRY(v4_flush_user_tlb_range) +SYM_TYPED_FUNC_START(v4_flush_user_tlb_range) vma_vm_mm ip, r2 act_mm r3 @ get current->active_mm eors r3, ip, r3 @ == mm ? @@ -40,6 +41,7 @@ ENTRY(v4_flush_user_tlb_range) cmp r0, r1 blo 1b ret lr +SYM_FUNC_END(v4_flush_user_tlb_range) /* * v4_flush_kern_tlb_range(start, end) @@ -50,10 +52,11 @@ ENTRY(v4_flush_user_tlb_range) * - start - virtual address (may not be aligned) * - end - virtual address (may not be aligned) */ +#ifdef CONFIG_CFI_CLANG +SYM_TYPED_FUNC_START(v4_flush_kern_tlb_range) + b .v4_flush_kern_tlb_range +SYM_FUNC_END(v4_flush_kern_tlb_range) +#else .globl v4_flush_kern_tlb_range .equ v4_flush_kern_tlb_range, .v4_flush_kern_tlb_range - - __INITDATA - - /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */ - define_tlb_functions v4, v4_tlb_flags +#endif diff --git a/arch/arm/mm/tlb-v4wb.S b/arch/arm/mm/tlb-v4wb.S index 9348bba7586a..04e46c359e75 100644 --- a/arch/arm/mm/tlb-v4wb.S +++ b/arch/arm/mm/tlb-v4wb.S @@ -11,6 +11,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/tlbflush.h> @@ -27,7 +28,7 @@ * - mm - mm_struct describing address space */ .align 5 -ENTRY(v4wb_flush_user_tlb_range) +SYM_TYPED_FUNC_START(v4wb_flush_user_tlb_range) vma_vm_mm ip, r2 act_mm r3 @ get current->active_mm eors r3, ip, r3 @ == mm ? @@ -43,6 +44,7 @@ ENTRY(v4wb_flush_user_tlb_range) cmp r0, r1 blo 1b ret lr +SYM_FUNC_END(v4wb_flush_user_tlb_range) /* * v4_flush_kern_tlb_range(start, end) @@ -53,7 +55,7 @@ ENTRY(v4wb_flush_user_tlb_range) * - start - virtual address (may not be aligned) * - end - virtual address (may not be aligned) */ -ENTRY(v4wb_flush_kern_tlb_range) +SYM_TYPED_FUNC_START(v4wb_flush_kern_tlb_range) mov r3, #0 mcr p15, 0, r3, c7, c10, 4 @ drain WB bic r0, r0, #0x0ff @@ -64,8 +66,4 @@ ENTRY(v4wb_flush_kern_tlb_range) cmp r0, r1 blo 1b ret lr - - __INITDATA - - /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */ - define_tlb_functions v4wb, v4wb_tlb_flags +SYM_FUNC_END(v4wb_flush_kern_tlb_range) diff --git a/arch/arm/mm/tlb-v4wbi.S b/arch/arm/mm/tlb-v4wbi.S index d4f9040a4111..502dfe5628a3 100644 --- a/arch/arm/mm/tlb-v4wbi.S +++ b/arch/arm/mm/tlb-v4wbi.S @@ -11,6 +11,7 @@ */ #include <linux/linkage.h> #include <linux/init.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/tlbflush.h> @@ -26,7 +27,7 @@ * - mm - mm_struct describing address space */ .align 5 -ENTRY(v4wbi_flush_user_tlb_range) +SYM_TYPED_FUNC_START(v4wbi_flush_user_tlb_range) vma_vm_mm ip, r2 act_mm r3 @ get current->active_mm eors r3, ip, r3 @ == mm ? @@ -43,8 +44,9 @@ ENTRY(v4wbi_flush_user_tlb_range) cmp r0, r1 blo 1b ret lr +SYM_FUNC_END(v4wbi_flush_user_tlb_range) -ENTRY(v4wbi_flush_kern_tlb_range) +SYM_TYPED_FUNC_START(v4wbi_flush_kern_tlb_range) mov r3, #0 mcr p15, 0, r3, c7, c10, 4 @ drain WB bic r0, r0, #0x0ff @@ -55,8 +57,4 @@ ENTRY(v4wbi_flush_kern_tlb_range) cmp r0, r1 blo 1b ret lr - - __INITDATA - - /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */ - define_tlb_functions v4wbi, v4wbi_tlb_flags +SYM_FUNC_END(v4wbi_flush_kern_tlb_range) diff --git a/arch/arm/mm/tlb-v6.S b/arch/arm/mm/tlb-v6.S index 5335b9687297..8256a67ac654 100644 --- a/arch/arm/mm/tlb-v6.S +++ b/arch/arm/mm/tlb-v6.S @@ -9,6 +9,7 @@ */ #include <linux/init.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/asm-offsets.h> #include <asm/assembler.h> #include <asm/page.h> @@ -17,6 +18,8 @@ #define HARVARD_TLB +.arch armv6 + /* * v6wbi_flush_user_tlb_range(start, end, vma) * @@ -24,13 +27,13 @@ * * - start - start address (may not be aligned) * - end - end address (exclusive, may not be aligned) - * - vma - vma_struct describing address range + * - vma - vm_area_struct describing address range * * It is assumed that: * - the "Invalidate single entry" instruction will invalidate * both the I and the D TLBs on Harvard-style TLBs */ -ENTRY(v6wbi_flush_user_tlb_range) +SYM_TYPED_FUNC_START(v6wbi_flush_user_tlb_range) vma_vm_mm r3, r2 @ get vma->vm_mm mov ip, #0 mmid r3, r3 @ get vm_mm->context.id @@ -54,6 +57,7 @@ ENTRY(v6wbi_flush_user_tlb_range) blo 1b mcr p15, 0, ip, c7, c10, 4 @ data synchronization barrier ret lr +SYM_FUNC_END(v6wbi_flush_user_tlb_range) /* * v6wbi_flush_kern_tlb_range(start,end) @@ -63,7 +67,7 @@ ENTRY(v6wbi_flush_user_tlb_range) * - start - start address (may not be aligned) * - end - end address (exclusive, may not be aligned) */ -ENTRY(v6wbi_flush_kern_tlb_range) +SYM_TYPED_FUNC_START(v6wbi_flush_kern_tlb_range) mov r2, #0 mcr p15, 0, r2, c7, c10, 4 @ drain write buffer mov r0, r0, lsr #PAGE_SHIFT @ align address @@ -83,8 +87,4 @@ ENTRY(v6wbi_flush_kern_tlb_range) mcr p15, 0, r2, c7, c10, 4 @ data synchronization barrier mcr p15, 0, r2, c7, c5, 4 @ prefetch flush (isb) ret lr - - __INIT - - /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */ - define_tlb_functions v6wbi, v6wbi_tlb_flags +SYM_FUNC_END(v6wbi_flush_kern_tlb_range) diff --git a/arch/arm/mm/tlb-v7.S b/arch/arm/mm/tlb-v7.S index 1bb28d7db567..f1aa0764a2cc 100644 --- a/arch/arm/mm/tlb-v7.S +++ b/arch/arm/mm/tlb-v7.S @@ -10,12 +10,15 @@ */ #include <linux/init.h> #include <linux/linkage.h> +#include <linux/cfi_types.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> #include <asm/page.h> #include <asm/tlbflush.h> #include "proc-macros.S" +.arch armv7-a + /* * v7wbi_flush_user_tlb_range(start, end, vma) * @@ -23,13 +26,13 @@ * * - start - start address (may not be aligned) * - end - end address (exclusive, may not be aligned) - * - vma - vma_struct describing address range + * - vma - vm_area_struct describing address range * * It is assumed that: * - the "Invalidate single entry" instruction will invalidate * both the I and the D TLBs on Harvard-style TLBs */ -ENTRY(v7wbi_flush_user_tlb_range) +SYM_TYPED_FUNC_START(v7wbi_flush_user_tlb_range) vma_vm_mm r3, r2 @ get vma->vm_mm mmid r3, r3 @ get vm_mm->context.id dsb ish @@ -55,7 +58,7 @@ ENTRY(v7wbi_flush_user_tlb_range) blo 1b dsb ish ret lr -ENDPROC(v7wbi_flush_user_tlb_range) +SYM_FUNC_END(v7wbi_flush_user_tlb_range) /* * v7wbi_flush_kern_tlb_range(start,end) @@ -65,7 +68,7 @@ ENDPROC(v7wbi_flush_user_tlb_range) * - start - start address (may not be aligned) * - end - end address (exclusive, may not be aligned) */ -ENTRY(v7wbi_flush_kern_tlb_range) +SYM_TYPED_FUNC_START(v7wbi_flush_kern_tlb_range) dsb ish mov r0, r0, lsr #PAGE_SHIFT @ align address mov r1, r1, lsr #PAGE_SHIFT @@ -84,9 +87,4 @@ ENTRY(v7wbi_flush_kern_tlb_range) dsb ish isb ret lr -ENDPROC(v7wbi_flush_kern_tlb_range) - - __INIT - - /* define struct cpu_tlb_fns (see <asm/tlbflush.h> and proc-macros.S) */ - define_tlb_functions v7wbi, v7wbi_tlb_flags_up, flags_smp=v7wbi_tlb_flags_smp +SYM_FUNC_END(v7wbi_flush_kern_tlb_range) diff --git a/arch/arm/mm/tlb.c b/arch/arm/mm/tlb.c new file mode 100644 index 000000000000..42359793120b --- /dev/null +++ b/arch/arm/mm/tlb.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright 2024 Google LLC +// Author: Ard Biesheuvel <ardb@google.com> + +#include <linux/types.h> +#include <asm/tlbflush.h> + +#ifdef CONFIG_CPU_TLB_V4WT +void v4_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); +void v4_flush_kern_tlb_range(unsigned long, unsigned long); + +struct cpu_tlb_fns v4_tlb_fns __initconst = { + .flush_user_range = v4_flush_user_tlb_range, + .flush_kern_range = v4_flush_kern_tlb_range, + .tlb_flags = v4_tlb_flags, +}; +#endif + +#ifdef CONFIG_CPU_TLB_V4WB +void v4wb_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); +void v4wb_flush_kern_tlb_range(unsigned long, unsigned long); + +struct cpu_tlb_fns v4wb_tlb_fns __initconst = { + .flush_user_range = v4wb_flush_user_tlb_range, + .flush_kern_range = v4wb_flush_kern_tlb_range, + .tlb_flags = v4wb_tlb_flags, +}; +#endif + +#if defined(CONFIG_CPU_TLB_V4WBI) || defined(CONFIG_CPU_TLB_FEROCEON) +void v4wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); +void v4wbi_flush_kern_tlb_range(unsigned long, unsigned long); + +struct cpu_tlb_fns v4wbi_tlb_fns __initconst = { + .flush_user_range = v4wbi_flush_user_tlb_range, + .flush_kern_range = v4wbi_flush_kern_tlb_range, + .tlb_flags = v4wbi_tlb_flags, +}; +#endif + +#ifdef CONFIG_CPU_TLB_V6 +void v6wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); +void v6wbi_flush_kern_tlb_range(unsigned long, unsigned long); + +struct cpu_tlb_fns v6wbi_tlb_fns __initconst = { + .flush_user_range = v6wbi_flush_user_tlb_range, + .flush_kern_range = v6wbi_flush_kern_tlb_range, + .tlb_flags = v6wbi_tlb_flags, +}; +#endif + +#ifdef CONFIG_CPU_TLB_V7 +void v7wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); +void v7wbi_flush_kern_tlb_range(unsigned long, unsigned long); + +struct cpu_tlb_fns v7wbi_tlb_fns __initconst = { + .flush_user_range = v7wbi_flush_user_tlb_range, + .flush_kern_range = v7wbi_flush_kern_tlb_range, + .tlb_flags = IS_ENABLED(CONFIG_SMP) ? v7wbi_tlb_flags_smp + : v7wbi_tlb_flags_up, +}; + +#ifdef CONFIG_SMP_ON_UP +/* This will be run-time patched so the offset better be right */ +static_assert(offsetof(struct cpu_tlb_fns, tlb_flags) == 8); + +asm(" .pushsection \".alt.smp.init\", \"a\" \n" \ + " .align 2 \n" \ + " .long v7wbi_tlb_fns + 8 - . \n" \ + " .long " __stringify(v7wbi_tlb_flags_up) " \n" \ + " .popsection \n"); +#endif +#endif + +#ifdef CONFIG_CPU_TLB_FA +void fa_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); +void fa_flush_kern_tlb_range(unsigned long, unsigned long); + +struct cpu_tlb_fns fa_tlb_fns __initconst = { + .flush_user_range = fa_flush_user_tlb_range, + .flush_kern_range = fa_flush_kern_tlb_range, + .tlb_flags = fa_tlb_flags, +}; +#endif |