summaryrefslogtreecommitdiff
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/arm/Kconfig10
-rw-r--r--arch/arm/Makefile2
-rw-r--r--arch/arm/boot/compressed/Makefile1
-rw-r--r--arch/arm/boot/compressed/head.S42
-rw-r--r--arch/arm/boot/compressed/string.c19
-rw-r--r--arch/arm/include/asm/fixmap.h2
-rw-r--r--arch/arm/include/asm/kasan.h33
-rw-r--r--arch/arm/include/asm/kasan_def.h81
-rw-r--r--arch/arm/include/asm/memory.h10
-rw-r--r--arch/arm/include/asm/pgalloc.h8
-rw-r--r--arch/arm/include/asm/pgtable-2level.h27
-rw-r--r--arch/arm/include/asm/prom.h4
-rw-r--r--arch/arm/include/asm/string.h26
-rw-r--r--arch/arm/include/asm/thread_info.h8
-rw-r--r--arch/arm/include/asm/uaccess-asm.h2
-rw-r--r--arch/arm/kernel/Makefile3
-rw-r--r--arch/arm/kernel/atags.h4
-rw-r--r--arch/arm/kernel/atags_parse.c6
-rw-r--r--arch/arm/kernel/devtree.c6
-rw-r--r--arch/arm/kernel/entry-armv.S28
-rw-r--r--arch/arm/kernel/entry-common.S9
-rw-r--r--arch/arm/kernel/head-common.S7
-rw-r--r--arch/arm/kernel/head.S9
-rw-r--r--arch/arm/kernel/hyp-stub.S6
-rw-r--r--arch/arm/kernel/iwmmxt.S89
-rw-r--r--arch/arm/kernel/iwmmxt.h47
-rw-r--r--arch/arm/kernel/setup.c25
-rw-r--r--arch/arm/kernel/smp.c15
-rw-r--r--arch/arm/kernel/unwind.c9
-rw-r--r--arch/arm/lib/memcpy.S4
-rw-r--r--arch/arm/lib/memmove.S6
-rw-r--r--arch/arm/lib/memset.S4
-rw-r--r--arch/arm/mm/Kconfig1
-rw-r--r--arch/arm/mm/Makefile5
-rw-r--r--arch/arm/mm/init.c1
-rw-r--r--arch/arm/mm/kasan_init.c291
-rw-r--r--arch/arm/mm/mmu.c38
-rw-r--r--arch/arm/mm/pgd.c16
-rw-r--r--arch/arm/mm/pv-fixup-asm.S4
-rw-r--r--arch/arm/vdso/Makefile2
-rw-r--r--arch/arm/vfp/entry.S17
-rw-r--r--arch/arm/vfp/vfphw.S5
-rw-r--r--arch/arm/vfp/vfpmodule.c72
43 files changed, 819 insertions, 185 deletions
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index fe2f17eb2b50..2c5eb5c2b310 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -67,6 +67,7 @@ config ARM
select HAVE_ARCH_BITREVERSE if (CPU_32v7M || CPU_32v7) && !CPU_32v6
select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL && !CPU_ENDIAN_BE32 && MMU
select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU
+ select HAVE_ARCH_KASAN if MMU && !XIP_KERNEL
select HAVE_ARCH_MMAP_RND_BITS if MMU
select HAVE_ARCH_SECCOMP
select HAVE_ARCH_SECCOMP_FILTER if AEABI && !OABI_COMPAT
@@ -1323,6 +1324,15 @@ config PAGE_OFFSET
default 0xB0000000 if VMSPLIT_3G_OPT
default 0xC0000000
+config KASAN_SHADOW_OFFSET
+ hex
+ depends on KASAN
+ default 0x1f000000 if PAGE_OFFSET=0x40000000
+ default 0x5f000000 if PAGE_OFFSET=0x80000000
+ default 0x9f000000 if PAGE_OFFSET=0xC0000000
+ default 0x8f000000 if PAGE_OFFSET=0xB0000000
+ default 0xffffffff
+
config NR_CPUS
int "Maximum number of CPUs (2-32)"
range 2 32
diff --git a/arch/arm/Makefile b/arch/arm/Makefile
index 4d76eab2b22d..3c0a64cefe52 100644
--- a/arch/arm/Makefile
+++ b/arch/arm/Makefile
@@ -10,7 +10,7 @@
#
# Copyright (C) 1995-2001 by Russell King
-LDFLAGS_vmlinux := --no-undefined -X --pic-veneer
+LDFLAGS_vmlinux := --no-undefined -X --pic-veneer -z norelro
ifeq ($(CONFIG_CPU_ENDIAN_BE8),y)
LDFLAGS_vmlinux += --be8
KBUILD_LDFLAGS_MODULE += --be8
diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile
index 47f001ca5499..a815b1ae990d 100644
--- a/arch/arm/boot/compressed/Makefile
+++ b/arch/arm/boot/compressed/Makefile
@@ -24,6 +24,7 @@ OBJS += hyp-stub.o
endif
GCOV_PROFILE := n
+KASAN_SANITIZE := n
# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in.
KCOV_INSTRUMENT := n
diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S
index 2e04ec5b5446..8d546cb10921 100644
--- a/arch/arm/boot/compressed/head.S
+++ b/arch/arm/boot/compressed/head.S
@@ -11,6 +11,12 @@
#include "efi-header.S"
+#ifdef __ARMEB__
+#define OF_DT_MAGIC 0xd00dfeed
+#else
+#define OF_DT_MAGIC 0xedfe0dd0
+#endif
+
AR_CLASS( .arch armv7-a )
M_CLASS( .arch armv7-m )
@@ -116,7 +122,7 @@
/*
* Debug print of the final appended DTB location
*/
- .macro dbgadtb, begin, end
+ .macro dbgadtb, begin, size
#ifdef DEBUG
kputc #'D'
kputc #'T'
@@ -129,7 +135,7 @@
kputc #'('
kputc #'0'
kputc #'x'
- kphex \end, 8 /* End of appended DTB */
+ kphex \size, 8 /* Size of appended DTB */
kputc #')'
kputc #'\n'
#endif
@@ -165,6 +171,16 @@
orr \res, \res, \tmp1, lsl #24
.endm
+ .macro be32tocpu, val, tmp
+#ifndef __ARMEB__
+ /* convert to little endian */
+ eor \tmp, \val, \val, ror #16
+ bic \tmp, \tmp, #0x00ff0000
+ mov \val, \val, ror #8
+ eor \val, \val, \tmp, lsr #8
+#endif
+ .endm
+
.section ".start", "ax"
/*
* sort out different calling conventions
@@ -325,11 +341,7 @@ restart: adr r0, LC1
*/
ldr lr, [r6, #0]
-#ifndef __ARMEB__
- ldr r1, =0xedfe0dd0 @ sig is 0xd00dfeed big endian
-#else
- ldr r1, =0xd00dfeed
-#endif
+ ldr r1, =OF_DT_MAGIC
cmp lr, r1
bne dtb_check_done @ not found
@@ -345,13 +357,7 @@ restart: adr r0, LC1
/* Get the initial DTB size */
ldr r5, [r6, #4]
-#ifndef __ARMEB__
- /* convert to little endian */
- eor r1, r5, r5, ror #16
- bic r1, r1, #0x00ff0000
- mov r5, r5, ror #8
- eor r5, r5, r1, lsr #8
-#endif
+ be32tocpu r5, r1
dbgadtb r6, r5
/* 50% DTB growth should be good enough */
add r5, r5, r5, lsr #1
@@ -403,13 +409,7 @@ restart: adr r0, LC1
/* Get the current DTB size */
ldr r5, [r6, #4]
-#ifndef __ARMEB__
- /* convert r5 (dtb size) to little endian */
- eor r1, r5, r5, ror #16
- bic r1, r1, #0x00ff0000
- mov r5, r5, ror #8
- eor r5, r5, r1, lsr #8
-#endif
+ be32tocpu r5, r1
/* preserve 64-bit alignment */
add r5, r5, #7
diff --git a/arch/arm/boot/compressed/string.c b/arch/arm/boot/compressed/string.c
index ade5079bebbf..8c0fa276d994 100644
--- a/arch/arm/boot/compressed/string.c
+++ b/arch/arm/boot/compressed/string.c
@@ -7,6 +7,25 @@
#include <linux/string.h>
+/*
+ * The decompressor is built without KASan but uses the same redirects as the
+ * rest of the kernel when CONFIG_KASAN is enabled, defining e.g. memcpy()
+ * to __memcpy() but since we are not linking with the main kernel string
+ * library in the decompressor, that will lead to link failures.
+ *
+ * Undefine KASan's versions, define the wrapped functions and alias them to
+ * the right names so that when e.g. __memcpy() appear in the code, it will
+ * still be linked to this local version of memcpy().
+ */
+#ifdef CONFIG_KASAN
+#undef memcpy
+#undef memmove
+#undef memset
+void *__memcpy(void *__dest, __const void *__src, size_t __n) __alias(memcpy);
+void *__memmove(void *__dest, __const void *__src, size_t count) __alias(memmove);
+void *__memset(void *s, int c, size_t count) __alias(memset);
+#endif
+
void *memcpy(void *__dest, __const void *__src, size_t __n)
{
int i = 0;
diff --git a/arch/arm/include/asm/fixmap.h b/arch/arm/include/asm/fixmap.h
index fc56fc3e1931..9575b404019c 100644
--- a/arch/arm/include/asm/fixmap.h
+++ b/arch/arm/include/asm/fixmap.h
@@ -2,7 +2,7 @@
#ifndef _ASM_FIXMAP_H
#define _ASM_FIXMAP_H
-#define FIXADDR_START 0xffc00000UL
+#define FIXADDR_START 0xffc80000UL
#define FIXADDR_END 0xfff00000UL
#define FIXADDR_TOP (FIXADDR_END - PAGE_SIZE)
diff --git a/arch/arm/include/asm/kasan.h b/arch/arm/include/asm/kasan.h
new file mode 100644
index 000000000000..303c35df3135
--- /dev/null
+++ b/arch/arm/include/asm/kasan.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * arch/arm/include/asm/kasan.h
+ *
+ * Copyright (c) 2015 Samsung Electronics Co., Ltd.
+ * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
+ *
+ */
+
+#ifndef __ASM_KASAN_H
+#define __ASM_KASAN_H
+
+#ifdef CONFIG_KASAN
+
+#include <asm/kasan_def.h>
+
+#define KASAN_SHADOW_SCALE_SHIFT 3
+
+/*
+ * The compiler uses a shadow offset assuming that addresses start
+ * from 0. Kernel addresses don't start from 0, so shadow
+ * for kernel really starts from 'compiler's shadow offset' +
+ * ('kernel address space start' >> KASAN_SHADOW_SCALE_SHIFT)
+ */
+
+asmlinkage void kasan_early_init(void);
+extern void kasan_init(void);
+
+#else
+static inline void kasan_init(void) { }
+#endif
+
+#endif
diff --git a/arch/arm/include/asm/kasan_def.h b/arch/arm/include/asm/kasan_def.h
new file mode 100644
index 000000000000..5739605aa7cf
--- /dev/null
+++ b/arch/arm/include/asm/kasan_def.h
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * arch/arm/include/asm/kasan_def.h
+ *
+ * Copyright (c) 2018 Huawei Technologies Co., Ltd.
+ *
+ * Author: Abbott Liu <liuwenliang@huawei.com>
+ */
+
+#ifndef __ASM_KASAN_DEF_H
+#define __ASM_KASAN_DEF_H
+
+#ifdef CONFIG_KASAN
+
+/*
+ * Define KASAN_SHADOW_OFFSET,KASAN_SHADOW_START and KASAN_SHADOW_END for
+ * the Arm kernel address sanitizer. We are "stealing" lowmem (the 4GB
+ * addressable by a 32bit architecture) out of the virtual address
+ * space to use as shadow memory for KASan as follows:
+ *
+ * +----+ 0xffffffff
+ * | | \
+ * | | |-> Static kernel image (vmlinux) BSS and page table
+ * | |/
+ * +----+ PAGE_OFFSET
+ * | | \
+ * | | |-> Loadable kernel modules virtual address space area
+ * | |/
+ * +----+ MODULES_VADDR = KASAN_SHADOW_END
+ * | | \
+ * | | |-> The shadow area of kernel virtual address.
+ * | |/
+ * +----+-> TASK_SIZE (start of kernel space) = KASAN_SHADOW_START the
+ * | |\ shadow address of MODULES_VADDR
+ * | | |
+ * | | |
+ * | | |-> The user space area in lowmem. The kernel address
+ * | | | sanitizer do not use this space, nor does it map it.
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | |/
+ * ------ 0
+ *
+ * 1) KASAN_SHADOW_START
+ * This value begins with the MODULE_VADDR's shadow address. It is the
+ * start of kernel virtual space. Since we have modules to load, we need
+ * to cover also that area with shadow memory so we can find memory
+ * bugs in modules.
+ *
+ * 2) KASAN_SHADOW_END
+ * This value is the 0x100000000's shadow address: the mapping that would
+ * be after the end of the kernel memory at 0xffffffff. It is the end of
+ * kernel address sanitizer shadow area. It is also the start of the
+ * module area.
+ *
+ * 3) KASAN_SHADOW_OFFSET:
+ * This value is used to map an address to the corresponding shadow
+ * address by the following formula:
+ *
+ * shadow_addr = (address >> 3) + KASAN_SHADOW_OFFSET;
+ *
+ * As you would expect, >> 3 is equal to dividing by 8, meaning each
+ * byte in the shadow memory covers 8 bytes of kernel memory, so one
+ * bit shadow memory per byte of kernel memory is used.
+ *
+ * The KASAN_SHADOW_OFFSET is provided in a Kconfig option depending
+ * on the VMSPLIT layout of the system: the kernel and userspace can
+ * split up lowmem in different ways according to needs, so we calculate
+ * the shadow offset depending on this.
+ */
+
+#define KASAN_SHADOW_SCALE_SHIFT 3
+#define KASAN_SHADOW_OFFSET _AC(CONFIG_KASAN_SHADOW_OFFSET, UL)
+#define KASAN_SHADOW_END ((UL(1) << (32 - KASAN_SHADOW_SCALE_SHIFT)) \
+ + KASAN_SHADOW_OFFSET)
+#define KASAN_SHADOW_START ((KASAN_SHADOW_END >> 3) + KASAN_SHADOW_OFFSET)
+
+#endif
+#endif
diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h
index 99035b5891ef..38a163f50130 100644
--- a/arch/arm/include/asm/memory.h
+++ b/arch/arm/include/asm/memory.h
@@ -18,6 +18,7 @@
#ifdef CONFIG_NEED_MACH_MEMORY_H
#include <mach/memory.h>
#endif
+#include <asm/kasan_def.h>
/* PAGE_OFFSET - the virtual address of the start of the kernel image */
#define PAGE_OFFSET UL(CONFIG_PAGE_OFFSET)
@@ -28,7 +29,11 @@
* TASK_SIZE - the maximum size of a user space task.
* TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area
*/
+#ifndef CONFIG_KASAN
#define TASK_SIZE (UL(CONFIG_PAGE_OFFSET) - UL(SZ_16M))
+#else
+#define TASK_SIZE (KASAN_SHADOW_START)
+#endif
#define TASK_UNMAPPED_BASE ALIGN(TASK_SIZE / 3, SZ_16M)
/*
@@ -67,6 +72,10 @@
*/
#define XIP_VIRT_ADDR(physaddr) (MODULES_VADDR + ((physaddr) & 0x000fffff))
+#define FDT_FIXED_BASE UL(0xff800000)
+#define FDT_FIXED_SIZE (2 * SECTION_SIZE)
+#define FDT_VIRT_BASE(physbase) ((void *)(FDT_FIXED_BASE | (physbase) % SECTION_SIZE))
+
#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
/*
* Allow 16MB-aligned ioremap pages
@@ -107,6 +116,7 @@ extern unsigned long vectors_base;
#define MODULES_VADDR PAGE_OFFSET
#define XIP_VIRT_ADDR(physaddr) (physaddr)
+#define FDT_VIRT_BASE(physbase) ((void *)(physbase))
#endif /* !CONFIG_MMU */
diff --git a/arch/arm/include/asm/pgalloc.h b/arch/arm/include/asm/pgalloc.h
index 15f4674715f8..fdee1f04f4f3 100644
--- a/arch/arm/include/asm/pgalloc.h
+++ b/arch/arm/include/asm/pgalloc.h
@@ -21,6 +21,7 @@
#define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_KERNEL))
#ifdef CONFIG_ARM_LPAE
+#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
@@ -28,14 +29,19 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
}
#else /* !CONFIG_ARM_LPAE */
+#define PGD_SIZE (PAGE_SIZE << 2)
/*
* Since we have only two-level page tables, these are trivial
*/
#define pmd_alloc_one(mm,addr) ({ BUG(); ((pmd_t *)2); })
#define pmd_free(mm, pmd) do { } while (0)
+#ifdef CONFIG_KASAN
+/* The KASan core unconditionally calls pud_populate() on all architectures */
+#define pud_populate(mm,pmd,pte) do { } while (0)
+#else
#define pud_populate(mm,pmd,pte) BUG()
-
+#endif
#endif /* CONFIG_ARM_LPAE */
extern pgd_t *pgd_alloc(struct mm_struct *mm);
diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h
index 3502c2f746ca..9d4f5eef410b 100644
--- a/arch/arm/include/asm/pgtable-2level.h
+++ b/arch/arm/include/asm/pgtable-2level.h
@@ -177,11 +177,28 @@
* the pud: the pud entry is never bad, always exists, and can't be set or
* cleared.
*/
-#define pud_none(pud) (0)
-#define pud_bad(pud) (0)
-#define pud_present(pud) (1)
-#define pud_clear(pudp) do { } while (0)
-#define set_pud(pud,pudp) do { } while (0)
+static inline int pud_none(pud_t pud)
+{
+ return 0;
+}
+
+static inline int pud_bad(pud_t pud)
+{
+ return 0;
+}
+
+static inline int pud_present(pud_t pud)
+{
+ return 1;
+}
+
+static inline void pud_clear(pud_t *pudp)
+{
+}
+
+static inline void set_pud(pud_t *pudp, pud_t pud)
+{
+}
static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
{
diff --git a/arch/arm/include/asm/prom.h b/arch/arm/include/asm/prom.h
index 1e36c40533c1..402e3f34c7ed 100644
--- a/arch/arm/include/asm/prom.h
+++ b/arch/arm/include/asm/prom.h
@@ -9,12 +9,12 @@
#ifdef CONFIG_OF
-extern const struct machine_desc *setup_machine_fdt(unsigned int dt_phys);
+extern const struct machine_desc *setup_machine_fdt(void *dt_virt);
extern void __init arm_dt_init_cpu_maps(void);
#else /* CONFIG_OF */
-static inline const struct machine_desc *setup_machine_fdt(unsigned int dt_phys)
+static inline const struct machine_desc *setup_machine_fdt(void *dt_virt)
{
return NULL;
}
diff --git a/arch/arm/include/asm/string.h b/arch/arm/include/asm/string.h
index 111a1d8a41dd..6c607c68f3ad 100644
--- a/arch/arm/include/asm/string.h
+++ b/arch/arm/include/asm/string.h
@@ -5,6 +5,9 @@
/*
* We don't do inline string functions, since the
* optimised inline asm versions are not small.
+ *
+ * The __underscore versions of some functions are for KASan to be able
+ * to replace them with instrumented versions.
*/
#define __HAVE_ARCH_STRRCHR
@@ -15,15 +18,18 @@ extern char * strchr(const char * s, int c);
#define __HAVE_ARCH_MEMCPY
extern void * memcpy(void *, const void *, __kernel_size_t);
+extern void *__memcpy(void *dest, const void *src, __kernel_size_t n);
#define __HAVE_ARCH_MEMMOVE
extern void * memmove(void *, const void *, __kernel_size_t);
+extern void *__memmove(void *dest, const void *src, __kernel_size_t n);
#define __HAVE_ARCH_MEMCHR
extern void * memchr(const void *, int, __kernel_size_t);
#define __HAVE_ARCH_MEMSET
extern void * memset(void *, int, __kernel_size_t);
+extern void *__memset(void *s, int c, __kernel_size_t n);
#define __HAVE_ARCH_MEMSET32
extern void *__memset32(uint32_t *, uint32_t v, __kernel_size_t);
@@ -39,4 +45,24 @@ static inline void *memset64(uint64_t *p, uint64_t v, __kernel_size_t n)
return __memset64(p, v, n * 8, v >> 32);
}
+/*
+ * For files that are not instrumented (e.g. mm/slub.c) we
+ * must use non-instrumented versions of the mem*
+ * functions named __memcpy() etc. All such kernel code has
+ * been tagged with KASAN_SANITIZE_file.o = n, which means
+ * that the address sanitization argument isn't passed to the
+ * compiler, and __SANITIZE_ADDRESS__ is not set. As a result
+ * these defines kick in.
+ */
+#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)
+#define memcpy(dst, src, len) __memcpy(dst, src, len)
+#define memmove(dst, src, len) __memmove(dst, src, len)
+#define memset(s, c, n) __memset(s, c, n)
+
+#ifndef __NO_FORTIFY
+#define __NO_FORTIFY /* FORTIFY_SOURCE uses __builtin_memcpy, etc. */
+#endif
+
+#endif
+
#endif
diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h
index 536b6b979f63..56fae7861fd3 100644
--- a/arch/arm/include/asm/thread_info.h
+++ b/arch/arm/include/asm/thread_info.h
@@ -13,7 +13,15 @@
#include <asm/fpstate.h>
#include <asm/page.h>
+#ifdef CONFIG_KASAN
+/*
+ * KASan uses a lot of extra stack space so the thread size order needs to
+ * be increased.
+ */
+#define THREAD_SIZE_ORDER 2
+#else
#define THREAD_SIZE_ORDER 1
+#endif
#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER)
#define THREAD_START_SP (THREAD_SIZE - 8)
diff --git a/arch/arm/include/asm/uaccess-asm.h b/arch/arm/include/asm/uaccess-asm.h
index 907571fd05c6..e6eb7a2aaf1e 100644
--- a/arch/arm/include/asm/uaccess-asm.h
+++ b/arch/arm/include/asm/uaccess-asm.h
@@ -85,7 +85,7 @@
*/
.macro uaccess_entry, tsk, tmp0, tmp1, tmp2, disable
ldr \tmp1, [\tsk, #TI_ADDR_LIMIT]
- mov \tmp2, #TASK_SIZE
+ ldr \tmp2, =TASK_SIZE
str \tmp2, [\tsk, #TI_ADDR_LIMIT]
DACR( mrc p15, 0, \tmp0, c3, c0, 0)
DACR( str \tmp0, [sp, #SVC_DACR])
diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile
index 89e5d864e923..f6431b46866e 100644
--- a/arch/arm/kernel/Makefile
+++ b/arch/arm/kernel/Makefile
@@ -21,6 +21,9 @@ obj-y := elf.o entry-common.o irq.o opcodes.o \
setup.o signal.o sigreturn_codes.o \
stacktrace.o sys_arm.o time.o traps.o
+KASAN_SANITIZE_stacktrace.o := n
+KASAN_SANITIZE_traps.o := n
+
ifneq ($(CONFIG_ARM_UNWIND),y)
obj-$(CONFIG_FRAME_POINTER) += return_address.o
endif
diff --git a/arch/arm/kernel/atags.h b/arch/arm/kernel/atags.h
index 067e12edc341..f2819c25b602 100644
--- a/arch/arm/kernel/atags.h
+++ b/arch/arm/kernel/atags.h
@@ -2,11 +2,11 @@
void convert_to_tag_list(struct tag *tags);
#ifdef CONFIG_ATAGS
-const struct machine_desc *setup_machine_tags(phys_addr_t __atags_pointer,
+const struct machine_desc *setup_machine_tags(void *__atags_vaddr,
unsigned int machine_nr);
#else
static inline const struct machine_desc * __init __noreturn
-setup_machine_tags(phys_addr_t __atags_pointer, unsigned int machine_nr)
+setup_machine_tags(void *__atags_vaddr, unsigned int machine_nr)
{
early_print("no ATAGS support: can't continue\n");
while (true);
diff --git a/arch/arm/kernel/atags_parse.c b/arch/arm/kernel/atags_parse.c
index 6c12d9fe694e..373b61f9a4f0 100644
--- a/arch/arm/kernel/atags_parse.c
+++ b/arch/arm/kernel/atags_parse.c
@@ -174,7 +174,7 @@ static void __init squash_mem_tags(struct tag *tag)
}
const struct machine_desc * __init
-setup_machine_tags(phys_addr_t __atags_pointer, unsigned int machine_nr)
+setup_machine_tags(void *atags_vaddr, unsigned int machine_nr)
{
struct tag *tags = (struct tag *)&default_tags;
const struct machine_desc *mdesc = NULL, *p;
@@ -195,8 +195,8 @@ setup_machine_tags(phys_addr_t __atags_pointer, unsigned int machine_nr)
if (!mdesc)
return NULL;
- if (__atags_pointer)
- tags = phys_to_virt(__atags_pointer);
+ if (atags_vaddr)
+ tags = atags_vaddr;
else if (mdesc->atag_offset)
tags = (void *)(PAGE_OFFSET + mdesc->atag_offset);
diff --git a/arch/arm/kernel/devtree.c b/arch/arm/kernel/devtree.c
index 7f0745a97e20..28311dd0fee6 100644
--- a/arch/arm/kernel/devtree.c
+++ b/arch/arm/kernel/devtree.c
@@ -203,12 +203,12 @@ static const void * __init arch_get_next_mach(const char *const **match)
/**
* setup_machine_fdt - Machine setup when an dtb was passed to the kernel
- * @dt_phys: physical address of dt blob
+ * @dt_virt: virtual address of dt blob
*
* If a dtb was passed to the kernel in r2, then use it to choose the
* correct machine_desc and to setup the system.
*/
-const struct machine_desc * __init setup_machine_fdt(unsigned int dt_phys)
+const struct machine_desc * __init setup_machine_fdt(void *dt_virt)
{
const struct machine_desc *mdesc, *mdesc_best = NULL;
@@ -221,7 +221,7 @@ const struct machine_desc * __init setup_machine_fdt(unsigned int dt_phys)
mdesc_best = &__mach_desc_GENERIC_DT;
#endif
- if (!dt_phys || !early_init_dt_verify(phys_to_virt(dt_phys)))
+ if (!dt_virt || !early_init_dt_verify(dt_virt))
return NULL;
mdesc = of_flat_dt_match_machine(mdesc_best, arch_get_next_mach);
diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S
index 55a47df04773..0ea8529a4872 100644
--- a/arch/arm/kernel/entry-armv.S
+++ b/arch/arm/kernel/entry-armv.S
@@ -252,31 +252,10 @@ __und_svc:
#else
svc_entry
#endif
- @
- @ call emulation code, which returns using r9 if it has emulated
- @ the instruction, or the more conventional lr if we are to treat
- @ this as a real undefined instruction
- @
- @ r0 - instruction
- @
-#ifndef CONFIG_THUMB2_KERNEL
- ldr r0, [r4, #-4]
-#else
- mov r1, #2
- ldrh r0, [r4, #-2] @ Thumb instruction at LR - 2
- cmp r0, #0xe800 @ 32-bit instruction if xx >= 0
- blo __und_svc_fault
- ldrh r9, [r4] @ bottom 16 bits
- add r4, r4, #2
- str r4, [sp, #S_PC]
- orr r0, r9, r0, lsl #16
-#endif
- badr r9, __und_svc_finish
- mov r2, r4
- bl call_fpe
mov r1, #4 @ PC correction to apply
-__und_svc_fault:
+ THUMB( tst r5, #PSR_T_BIT ) @ exception taken in Thumb mode?
+ THUMB( movne r1, #2 ) @ if so, fix up PC correction
mov r0, sp @ struct pt_regs *regs
bl __und_fault
@@ -427,7 +406,8 @@ ENDPROC(__fiq_abt)
@ if it was interrupted in a critical region. Here we
@ perform a quick test inline since it should be false
@ 99.9999% of the time. The rest is done out of line.
- cmp r4, #TASK_SIZE
+ ldr r0, =TASK_SIZE
+ cmp r4, r0
blhs kuser_cmpxchg64_fixup
#endif
#endif
diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S
index 271cb8a1eba1..fee279e28a72 100644
--- a/arch/arm/kernel/entry-common.S
+++ b/arch/arm/kernel/entry-common.S
@@ -50,7 +50,8 @@ __ret_fast_syscall:
UNWIND(.cantunwind )
disable_irq_notrace @ disable interrupts
ldr r2, [tsk, #TI_ADDR_LIMIT]
- cmp r2, #TASK_SIZE
+ ldr r1, =TASK_SIZE
+ cmp r2, r1
blne addr_limit_check_failed
ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing
tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
@@ -87,7 +88,8 @@ __ret_fast_syscall:
#endif
disable_irq_notrace @ disable interrupts
ldr r2, [tsk, #TI_ADDR_LIMIT]
- cmp r2, #TASK_SIZE
+ ldr r1, =TASK_SIZE
+ cmp r2, r1
blne addr_limit_check_failed
ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing
tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
@@ -128,7 +130,8 @@ ret_slow_syscall:
disable_irq_notrace @ disable interrupts
ENTRY(ret_to_user_from_irq)
ldr r2, [tsk, #TI_ADDR_LIMIT]
- cmp r2, #TASK_SIZE
+ ldr r1, =TASK_SIZE
+ cmp r2, r1
blne addr_limit_check_failed
ldr r1, [tsk, #TI_FLAGS]
tst r1, #_TIF_WORK_MASK
diff --git a/arch/arm/kernel/head-common.S b/arch/arm/kernel/head-common.S
index 4a3982812a40..89c80154b9ef 100644
--- a/arch/arm/kernel/head-common.S
+++ b/arch/arm/kernel/head-common.S
@@ -95,7 +95,7 @@ __mmap_switched:
THUMB( ldmia r4!, {r0, r1, r2, r3} )
THUMB( mov sp, r3 )
sub r2, r2, r1
- bl memcpy @ copy .data to RAM
+ bl __memcpy @ copy .data to RAM
#endif
ARM( ldmia r4!, {r0, r1, sp} )
@@ -103,7 +103,7 @@ __mmap_switched:
THUMB( mov sp, r3 )
sub r2, r1, r0
mov r1, #0
- bl memset @ clear .bss
+ bl __memset @ clear .bss
ldmia r4, {r0, r1, r2, r3}
str r9, [r0] @ Save processor ID
@@ -111,6 +111,9 @@ __mmap_switched:
str r8, [r2] @ Save atags pointer
cmp r3, #0
strne r10, [r3] @ Save control register values
+#ifdef CONFIG_KASAN
+ bl kasan_early_init
+#endif
mov lr, #0
b start_kernel
ENDPROC(__mmap_switched)
diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S
index f8904227e7fd..26ffd462a975 100644
--- a/arch/arm/kernel/head.S
+++ b/arch/arm/kernel/head.S
@@ -274,11 +274,10 @@ __create_page_tables:
* We map 2 sections in case the ATAGs/DTB crosses a section boundary.
*/
mov r0, r2, lsr #SECTION_SHIFT
- movs r0, r0, lsl #SECTION_SHIFT
- subne r3, r0, r8
- addne r3, r3, #PAGE_OFFSET
- addne r3, r4, r3, lsr #(SECTION_SHIFT - PMD_ORDER)
- orrne r6, r7, r0
+ cmp r2, #0
+ ldrne r3, =FDT_FIXED_BASE >> (SECTION_SHIFT - PMD_ORDER)
+ addne r3, r3, r4
+ orrne r6, r7, r0, lsl #SECTION_SHIFT
strne r6, [r3], #1 << PMD_ORDER
addne r6, r6, #1 << SECTION_SHIFT
strne r6, [r3]
diff --git a/arch/arm/kernel/hyp-stub.S b/arch/arm/kernel/hyp-stub.S
index 26d8e03b1dd3..327f63ec42c6 100644
--- a/arch/arm/kernel/hyp-stub.S
+++ b/arch/arm/kernel/hyp-stub.S
@@ -228,12 +228,6 @@ ENTRY(__hyp_soft_restart)
ret lr
ENDPROC(__hyp_soft_restart)
-#ifndef ZIMAGE
-.align 2
-.L__boot_cpu_mode_offset:
- .long __boot_cpu_mode - .
-#endif
-
.align 5
ENTRY(__hyp_stub_vectors)
__hyp_stub_reset: W(b) .
diff --git a/arch/arm/kernel/iwmmxt.S b/arch/arm/kernel/iwmmxt.S
index 0dcae787b004..d2b4ac06e4ed 100644
--- a/arch/arm/kernel/iwmmxt.S
+++ b/arch/arm/kernel/iwmmxt.S
@@ -16,6 +16,7 @@
#include <asm/thread_info.h>
#include <asm/asm-offsets.h>
#include <asm/assembler.h>
+#include "iwmmxt.h"
#if defined(CONFIG_CPU_PJ4) || defined(CONFIG_CPU_PJ4B)
#define PJ4(code...) code
@@ -113,33 +114,33 @@ concan_save:
concan_dump:
- wstrw wCSSF, [r1, #MMX_WCSSF]
- wstrw wCASF, [r1, #MMX_WCASF]
- wstrw wCGR0, [r1, #MMX_WCGR0]
- wstrw wCGR1, [r1, #MMX_WCGR1]
- wstrw wCGR2, [r1, #MMX_WCGR2]
- wstrw wCGR3, [r1, #MMX_WCGR3]
+ wstrw wCSSF, r1, MMX_WCSSF
+ wstrw wCASF, r1, MMX_WCASF
+ wstrw wCGR0, r1, MMX_WCGR0
+ wstrw wCGR1, r1, MMX_WCGR1
+ wstrw wCGR2, r1, MMX_WCGR2
+ wstrw wCGR3, r1, MMX_WCGR3
1: @ MUP? wRn
tst r2, #0x2
beq 2f
- wstrd wR0, [r1, #MMX_WR0]
- wstrd wR1, [r1, #MMX_WR1]
- wstrd wR2, [r1, #MMX_WR2]
- wstrd wR3, [r1, #MMX_WR3]
- wstrd wR4, [r1, #MMX_WR4]
- wstrd wR5, [r1, #MMX_WR5]
- wstrd wR6, [r1, #MMX_WR6]
- wstrd wR7, [r1, #MMX_WR7]
- wstrd wR8, [r1, #MMX_WR8]
- wstrd wR9, [r1, #MMX_WR9]
- wstrd wR10, [r1, #MMX_WR10]
- wstrd wR11, [r1, #MMX_WR11]
- wstrd wR12, [r1, #MMX_WR12]
- wstrd wR13, [r1, #MMX_WR13]
- wstrd wR14, [r1, #MMX_WR14]
- wstrd wR15, [r1, #MMX_WR15]
+ wstrd wR0, r1, MMX_WR0
+ wstrd wR1, r1, MMX_WR1
+ wstrd wR2, r1, MMX_WR2
+ wstrd wR3, r1, MMX_WR3
+ wstrd wR4, r1, MMX_WR4
+ wstrd wR5, r1, MMX_WR5
+ wstrd wR6, r1, MMX_WR6
+ wstrd wR7, r1, MMX_WR7
+ wstrd wR8, r1, MMX_WR8
+ wstrd wR9, r1, MMX_WR9
+ wstrd wR10, r1, MMX_WR10
+ wstrd wR11, r1, MMX_WR11
+ wstrd wR12, r1, MMX_WR12
+ wstrd wR13, r1, MMX_WR13
+ wstrd wR14, r1, MMX_WR14
+ wstrd wR15, r1, MMX_WR15
2: teq r0, #0 @ anything to load?
reteq lr @ if not, return
@@ -147,30 +148,30 @@ concan_dump:
concan_load:
@ Load wRn
- wldrd wR0, [r0, #MMX_WR0]
- wldrd wR1, [r0, #MMX_WR1]
- wldrd wR2, [r0, #MMX_WR2]
- wldrd wR3, [r0, #MMX_WR3]
- wldrd wR4, [r0, #MMX_WR4]
- wldrd wR5, [r0, #MMX_WR5]
- wldrd wR6, [r0, #MMX_WR6]
- wldrd wR7, [r0, #MMX_WR7]
- wldrd wR8, [r0, #MMX_WR8]
- wldrd wR9, [r0, #MMX_WR9]
- wldrd wR10, [r0, #MMX_WR10]
- wldrd wR11, [r0, #MMX_WR11]
- wldrd wR12, [r0, #MMX_WR12]
- wldrd wR13, [r0, #MMX_WR13]
- wldrd wR14, [r0, #MMX_WR14]
- wldrd wR15, [r0, #MMX_WR15]
+ wldrd wR0, r0, MMX_WR0
+ wldrd wR1, r0, MMX_WR1
+ wldrd wR2, r0, MMX_WR2
+ wldrd wR3, r0, MMX_WR3
+ wldrd wR4, r0, MMX_WR4
+ wldrd wR5, r0, MMX_WR5
+ wldrd wR6, r0, MMX_WR6
+ wldrd wR7, r0, MMX_WR7
+ wldrd wR8, r0, MMX_WR8
+ wldrd wR9, r0, MMX_WR9
+ wldrd wR10, r0, MMX_WR10
+ wldrd wR11, r0, MMX_WR11
+ wldrd wR12, r0, MMX_WR12
+ wldrd wR13, r0, MMX_WR13
+ wldrd wR14, r0, MMX_WR14
+ wldrd wR15, r0, MMX_WR15
@ Load wCx
- wldrw wCSSF, [r0, #MMX_WCSSF]
- wldrw wCASF, [r0, #MMX_WCASF]
- wldrw wCGR0, [r0, #MMX_WCGR0]
- wldrw wCGR1, [r0, #MMX_WCGR1]
- wldrw wCGR2, [r0, #MMX_WCGR2]
- wldrw wCGR3, [r0, #MMX_WCGR3]
+ wldrw wCSSF, r0, MMX_WCSSF
+ wldrw wCASF, r0, MMX_WCASF
+ wldrw wCGR0, r0, MMX_WCGR0
+ wldrw wCGR1, r0, MMX_WCGR1
+ wldrw wCGR2, r0, MMX_WCGR2
+ wldrw wCGR3, r0, MMX_WCGR3
@ clear CUP/MUP (only if r1 != 0)
teq r1, #0
diff --git a/arch/arm/kernel/iwmmxt.h b/arch/arm/kernel/iwmmxt.h
new file mode 100644
index 000000000000..fb627286f5bb
--- /dev/null
+++ b/arch/arm/kernel/iwmmxt.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __IWMMXT_H__
+#define __IWMMXT_H__
+
+.irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+.set .LwR\b, \b
+.set .Lr\b, \b
+.endr
+
+.set .LwCSSF, 0x2
+.set .LwCASF, 0x3
+.set .LwCGR0, 0x8
+.set .LwCGR1, 0x9
+.set .LwCGR2, 0xa
+.set .LwCGR3, 0xb
+
+.macro wldrd, reg:req, base:req, offset:req
+.inst 0xedd00100 | (.L\reg << 12) | (.L\base << 16) | (\offset >> 2)
+.endm
+
+.macro wldrw, reg:req, base:req, offset:req
+.inst 0xfd900100 | (.L\reg << 12) | (.L\base << 16) | (\offset >> 2)
+.endm
+
+.macro wstrd, reg:req, base:req, offset:req
+.inst 0xedc00100 | (.L\reg << 12) | (.L\base << 16) | (\offset >> 2)
+.endm
+
+.macro wstrw, reg:req, base:req, offset:req
+.inst 0xfd800100 | (.L\reg << 12) | (.L\base << 16) | (\offset >> 2)
+.endm
+
+#ifdef __clang__
+
+#define wCon c1
+
+.macro tmrc, dest:req, control:req
+mrc p1, 0, \dest, \control, c0, 0
+.endm
+
+.macro tmcr, control:req, src:req
+mcr p1, 0, \src, \control, c0, 0
+.endm
+#endif
+
+#endif
diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c
index 3f65d0ac9f63..1a5edf562e85 100644
--- a/arch/arm/kernel/setup.c
+++ b/arch/arm/kernel/setup.c
@@ -18,6 +18,7 @@
#include <linux/of_platform.h>
#include <linux/init.h>
#include <linux/kexec.h>
+#include <linux/libfdt.h>
#include <linux/of_fdt.h>
#include <linux/cpu.h>
#include <linux/interrupt.h>
@@ -58,6 +59,7 @@
#include <asm/unwind.h>
#include <asm/memblock.h>
#include <asm/virt.h>
+#include <asm/kasan.h>
#include "atags.h"
@@ -763,7 +765,7 @@ int __init arm_add_memory(u64 start, u64 size)
#ifndef CONFIG_PHYS_ADDR_T_64BIT
if (aligned_start > ULONG_MAX) {
pr_crit("Ignoring memory at 0x%08llx outside 32-bit physical address space\n",
- (long long)start);
+ start);
return -EINVAL;
}
@@ -1081,19 +1083,27 @@ void __init hyp_mode_check(void)
void __init setup_arch(char **cmdline_p)
{
- const struct machine_desc *mdesc;
+ const struct machine_desc *mdesc = NULL;
+ void *atags_vaddr = NULL;
+
+ if (__atags_pointer)
+ atags_vaddr = FDT_VIRT_BASE(__atags_pointer);
setup_processor();
- mdesc = setup_machine_fdt(__atags_pointer);
+ if (atags_vaddr) {
+ mdesc = setup_machine_fdt(atags_vaddr);
+ if (mdesc)
+ memblock_reserve(__atags_pointer,
+ fdt_totalsize(atags_vaddr));
+ }
if (!mdesc)
- mdesc = setup_machine_tags(__atags_pointer, __machine_arch_type);
+ mdesc = setup_machine_tags(atags_vaddr, __machine_arch_type);
if (!mdesc) {
early_print("\nError: invalid dtb and unrecognized/unsupported machine ID\n");
early_print(" r1=0x%08x, r2=0x%08x\n", __machine_arch_type,
__atags_pointer);
if (__atags_pointer)
- early_print(" r2[]=%*ph\n", 16,
- phys_to_virt(__atags_pointer));
+ early_print(" r2[]=%*ph\n", 16, atags_vaddr);
dump_machine_table();
}
@@ -1126,7 +1136,7 @@ void __init setup_arch(char **cmdline_p)
efi_init();
/*
* Make sure the calculation for lowmem/highmem is set appropriately
- * before reserving/allocating any mmeory
+ * before reserving/allocating any memory
*/
adjust_lowmem_bounds();
arm_memblock_init(mdesc);
@@ -1136,6 +1146,7 @@ void __init setup_arch(char **cmdline_p)
early_ioremap_reset();
paging_init(mdesc);
+ kasan_init();
request_standard_resources(mdesc);
if (mdesc->restart)
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index 48099c6e1e4a..6ab2b0ad5f40 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -524,14 +524,13 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
}
static const char *ipi_types[NR_IPI] __tracepoint_string = {
-#define S(x,s) [x] = s
- S(IPI_WAKEUP, "CPU wakeup interrupts"),
- S(IPI_TIMER, "Timer broadcast interrupts"),
- S(IPI_RESCHEDULE, "Rescheduling interrupts"),
- S(IPI_CALL_FUNC, "Function call interrupts"),
- S(IPI_CPU_STOP, "CPU stop interrupts"),
- S(IPI_IRQ_WORK, "IRQ work interrupts"),
- S(IPI_COMPLETION, "completion interrupts"),
+ [IPI_WAKEUP] = "CPU wakeup interrupts",
+ [IPI_TIMER] = "Timer broadcast interrupts",
+ [IPI_RESCHEDULE] = "Rescheduling interrupts",
+ [IPI_CALL_FUNC] = "Function call interrupts",
+ [IPI_CPU_STOP] = "CPU stop interrupts",
+ [IPI_IRQ_WORK] = "IRQ work interrupts",
+ [IPI_COMPLETION] = "completion interrupts",
};
static void smp_cross_call(const struct cpumask *target, unsigned int ipinr);
diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c
index d2bd0df2318d..59fdf257bf8b 100644
--- a/arch/arm/kernel/unwind.c
+++ b/arch/arm/kernel/unwind.c
@@ -18,9 +18,6 @@
#warning Your compiler does not have EABI support.
#warning ARM unwind is known to compile only with EABI compilers.
#warning Change compiler or disable ARM_UNWIND option.
-#elif (__GNUC__ == 4 && __GNUC_MINOR__ <= 2) && !defined(__clang__)
-#warning Your compiler is too buggy; it is known to not compile ARM unwind support.
-#warning Change compiler or disable ARM_UNWIND option.
#endif
#endif /* __CHECKER__ */
@@ -236,7 +233,11 @@ static int unwind_pop_register(struct unwind_ctrl_block *ctrl,
if (*vsp >= (unsigned long *)ctrl->sp_high)
return -URC_FAILURE;
- ctrl->vrs[reg] = *(*vsp)++;
+ /* Use READ_ONCE_NOCHECK here to avoid this memory access
+ * from being tracked by KASAN.
+ */
+ ctrl->vrs[reg] = READ_ONCE_NOCHECK(*(*vsp));
+ (*vsp)++;
return URC_OK;
}
diff --git a/arch/arm/lib/memcpy.S b/arch/arm/lib/memcpy.S
index 09a333153dc6..e4caf48c089f 100644
--- a/arch/arm/lib/memcpy.S
+++ b/arch/arm/lib/memcpy.S
@@ -58,10 +58,12 @@
/* Prototype: void *memcpy(void *dest, const void *src, size_t n); */
+ENTRY(__memcpy)
ENTRY(mmiocpy)
-ENTRY(memcpy)
+WEAK(memcpy)
#include "copy_template.S"
ENDPROC(memcpy)
ENDPROC(mmiocpy)
+ENDPROC(__memcpy)
diff --git a/arch/arm/lib/memmove.S b/arch/arm/lib/memmove.S
index b50e5770fb44..6fecc12a1f51 100644
--- a/arch/arm/lib/memmove.S
+++ b/arch/arm/lib/memmove.S
@@ -24,12 +24,13 @@
* occurring in the opposite direction.
*/
-ENTRY(memmove)
+ENTRY(__memmove)
+WEAK(memmove)
UNWIND( .fnstart )
subs ip, r0, r1
cmphi r2, ip
- bls memcpy
+ bls __memcpy
stmfd sp!, {r0, r4, lr}
UNWIND( .fnend )
@@ -222,3 +223,4 @@ ENTRY(memmove)
18: backward_copy_shift push=24 pull=8
ENDPROC(memmove)
+ENDPROC(__memmove)
diff --git a/arch/arm/lib/memset.S b/arch/arm/lib/memset.S
index 6ca4535c47fb..9817cb258c1a 100644
--- a/arch/arm/lib/memset.S
+++ b/arch/arm/lib/memset.S
@@ -13,8 +13,9 @@
.text
.align 5
+ENTRY(__memset)
ENTRY(mmioset)
-ENTRY(memset)
+WEAK(memset)
UNWIND( .fnstart )
ands r3, r0, #3 @ 1 unaligned?
mov ip, r0 @ preserve r0 as return value
@@ -132,6 +133,7 @@ UNWIND( .fnstart )
UNWIND( .fnend )
ENDPROC(memset)
ENDPROC(mmioset)
+ENDPROC(__memset)
ENTRY(__memset32)
UNWIND( .fnstart )
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 65e4482e3849..02692fbe2db5 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -743,6 +743,7 @@ config SWP_EMULATE
config CPU_BIG_ENDIAN
bool "Build big-endian kernel"
depends on ARCH_SUPPORTS_BIG_ENDIAN
+ depends on !LD_IS_LLD
help
Say Y if you plan on running a kernel in big-endian mode.
Note that your board must be properly built and your board
diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile
index 7cb1699fbfc4..4536159bc8fa 100644
--- a/arch/arm/mm/Makefile
+++ b/arch/arm/mm/Makefile
@@ -7,6 +7,7 @@ obj-y := extable.o fault.o init.o iomap.o
obj-y += dma-mapping$(MMUEXT).o
obj-$(CONFIG_MMU) += fault-armv.o flush.o idmap.o ioremap.o \
mmap.o pgd.o mmu.o pageattr.o
+KASAN_SANITIZE_mmu.o := n
ifneq ($(CONFIG_MMU),y)
obj-y += nommu.o
@@ -16,6 +17,7 @@ endif
obj-$(CONFIG_ARM_PTDUMP_CORE) += dump.o
obj-$(CONFIG_ARM_PTDUMP_DEBUGFS) += ptdump_debugfs.o
obj-$(CONFIG_MODULES) += proc-syms.o
+KASAN_SANITIZE_physaddr.o := n
obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o
obj-$(CONFIG_ALIGNMENT_TRAP) += alignment.o
@@ -111,3 +113,6 @@ obj-$(CONFIG_CACHE_L2X0_PMU) += cache-l2x0-pmu.o
obj-$(CONFIG_CACHE_XSC3L2) += cache-xsc3l2.o
obj-$(CONFIG_CACHE_TAUROS2) += cache-tauros2.o
obj-$(CONFIG_CACHE_UNIPHIER) += cache-uniphier.o
+
+KASAN_SANITIZE_kasan_init.o := n
+obj-$(CONFIG_KASAN) += kasan_init.o
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index d57112a276f5..a391804c7ce3 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -223,7 +223,6 @@ void __init arm_memblock_init(const struct machine_desc *mdesc)
if (mdesc->reserve)
mdesc->reserve();
- early_init_fdt_reserve_self();
early_init_fdt_scan_reserved_mem();
/* reserve memory for DMA contiguous allocations */
diff --git a/arch/arm/mm/kasan_init.c b/arch/arm/mm/kasan_init.c
new file mode 100644
index 000000000000..9c348042a724
--- /dev/null
+++ b/arch/arm/mm/kasan_init.c
@@ -0,0 +1,291 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * This file contains kasan initialization code for ARM.
+ *
+ * Copyright (c) 2018 Samsung Electronics Co., Ltd.
+ * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
+ * Author: Linus Walleij <linus.walleij@linaro.org>
+ */
+
+#define pr_fmt(fmt) "kasan: " fmt
+#include <linux/kasan.h>
+#include <linux/kernel.h>
+#include <linux/memblock.h>
+#include <linux/sched/task.h>
+#include <linux/start_kernel.h>
+#include <linux/pgtable.h>
+#include <asm/cputype.h>
+#include <asm/highmem.h>
+#include <asm/mach/map.h>
+#include <asm/memory.h>
+#include <asm/page.h>
+#include <asm/pgalloc.h>
+#include <asm/procinfo.h>
+#include <asm/proc-fns.h>
+
+#include "mm.h"
+
+static pgd_t tmp_pgd_table[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);
+
+pmd_t tmp_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
+
+static __init void *kasan_alloc_block(size_t size)
+{
+ return memblock_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS),
+ MEMBLOCK_ALLOC_KASAN, NUMA_NO_NODE);
+}
+
+static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,
+ unsigned long end, bool early)
+{
+ unsigned long next;
+ pte_t *ptep = pte_offset_kernel(pmdp, addr);
+
+ do {
+ pte_t entry;
+ void *p;
+
+ next = addr + PAGE_SIZE;
+
+ if (!early) {
+ if (!pte_none(READ_ONCE(*ptep)))
+ continue;
+
+ p = kasan_alloc_block(PAGE_SIZE);
+ if (!p) {
+ panic("%s failed to allocate shadow page for address 0x%lx\n",
+ __func__, addr);
+ return;
+ }
+ memset(p, KASAN_SHADOW_INIT, PAGE_SIZE);
+ entry = pfn_pte(virt_to_pfn(p),
+ __pgprot(pgprot_val(PAGE_KERNEL)));
+ } else if (pte_none(READ_ONCE(*ptep))) {
+ /*
+ * The early shadow memory is mapping all KASan
+ * operations to one and the same page in memory,
+ * "kasan_early_shadow_page" so that the instrumentation
+ * will work on a scratch area until we can set up the
+ * proper KASan shadow memory.
+ */
+ entry = pfn_pte(virt_to_pfn(kasan_early_shadow_page),
+ __pgprot(_L_PTE_DEFAULT | L_PTE_DIRTY | L_PTE_XN));
+ } else {
+ /*
+ * Early shadow mappings are PMD_SIZE aligned, so if the
+ * first entry is already set, they must all be set.
+ */
+ return;
+ }
+
+ set_pte_at(&init_mm, addr, ptep, entry);
+ } while (ptep++, addr = next, addr != end);
+}
+
+/*
+ * The pmd (page middle directory) is only used on LPAE
+ */
+static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,
+ unsigned long end, bool early)
+{
+ unsigned long next;
+ pmd_t *pmdp = pmd_offset(pudp, addr);
+
+ do {
+ if (pmd_none(*pmdp)) {
+ /*
+ * We attempt to allocate a shadow block for the PMDs
+ * used by the PTEs for this address if it isn't already
+ * allocated.
+ */
+ void *p = early ? kasan_early_shadow_pte :
+ kasan_alloc_block(PAGE_SIZE);
+
+ if (!p) {
+ panic("%s failed to allocate shadow block for address 0x%lx\n",
+ __func__, addr);
+ return;
+ }
+ pmd_populate_kernel(&init_mm, pmdp, p);
+ flush_pmd_entry(pmdp);
+ }
+
+ next = pmd_addr_end(addr, end);
+ kasan_pte_populate(pmdp, addr, next, early);
+ } while (pmdp++, addr = next, addr != end);
+}
+
+static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
+ bool early)
+{
+ unsigned long next;
+ pgd_t *pgdp;
+ p4d_t *p4dp;
+ pud_t *pudp;
+
+ pgdp = pgd_offset_k(addr);
+
+ do {
+ /*
+ * Allocate and populate the shadow block of p4d folded into
+ * pud folded into pmd if it doesn't already exist
+ */
+ if (!early && pgd_none(*pgdp)) {
+ void *p = kasan_alloc_block(PAGE_SIZE);
+
+ if (!p) {
+ panic("%s failed to allocate shadow block for address 0x%lx\n",
+ __func__, addr);
+ return;
+ }
+ pgd_populate(&init_mm, pgdp, p);
+ }
+
+ next = pgd_addr_end(addr, end);
+ /*
+ * We just immediately jump over the p4d and pud page
+ * directories since we believe ARM32 will never gain four
+ * nor five level page tables.
+ */
+ p4dp = p4d_offset(pgdp, addr);
+ pudp = pud_offset(p4dp, addr);
+
+ kasan_pmd_populate(pudp, addr, next, early);
+ } while (pgdp++, addr = next, addr != end);
+}
+
+extern struct proc_info_list *lookup_processor_type(unsigned int);
+
+void __init kasan_early_init(void)
+{
+ struct proc_info_list *list;
+
+ /*
+ * locate processor in the list of supported processor
+ * types. The linker builds this table for us from the
+ * entries in arch/arm/mm/proc-*.S
+ */
+ list = lookup_processor_type(read_cpuid_id());
+ if (list) {
+#ifdef MULTI_CPU
+ processor = *list->proc;
+#endif
+ }
+
+ BUILD_BUG_ON((KASAN_SHADOW_END - (1UL << 29)) != KASAN_SHADOW_OFFSET);
+ /*
+ * We walk the page table and set all of the shadow memory to point
+ * to the scratch page.
+ */
+ kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, true);
+}
+
+static void __init clear_pgds(unsigned long start,
+ unsigned long end)
+{
+ for (; start && start < end; start += PMD_SIZE)
+ pmd_clear(pmd_off_k(start));
+}
+
+static int __init create_mapping(void *start, void *end)
+{
+ void *shadow_start, *shadow_end;
+
+ shadow_start = kasan_mem_to_shadow(start);
+ shadow_end = kasan_mem_to_shadow(end);
+
+ pr_info("Mapping kernel virtual memory block: %px-%px at shadow: %px-%px\n",
+ start, end, shadow_start, shadow_end);
+
+ kasan_pgd_populate((unsigned long)shadow_start & PAGE_MASK,
+ PAGE_ALIGN((unsigned long)shadow_end), false);
+ return 0;
+}
+
+void __init kasan_init(void)
+{
+ phys_addr_t pa_start, pa_end;
+ u64 i;
+
+ /*
+ * We are going to perform proper setup of shadow memory.
+ *
+ * At first we should unmap early shadow (clear_pgds() call bellow).
+ * However, instrumented code can't execute without shadow memory.
+ *
+ * To keep the early shadow memory MMU tables around while setting up
+ * the proper shadow memory, we copy swapper_pg_dir (the initial page
+ * table) to tmp_pgd_table and use that to keep the early shadow memory
+ * mapped until the full shadow setup is finished. Then we swap back
+ * to the proper swapper_pg_dir.
+ */
+
+ memcpy(tmp_pgd_table, swapper_pg_dir, sizeof(tmp_pgd_table));
+#ifdef CONFIG_ARM_LPAE
+ /* We need to be in the same PGD or this won't work */
+ BUILD_BUG_ON(pgd_index(KASAN_SHADOW_START) !=
+ pgd_index(KASAN_SHADOW_END));
+ memcpy(tmp_pmd_table,
+ pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_START)),
+ sizeof(tmp_pmd_table));
+ set_pgd(&tmp_pgd_table[pgd_index(KASAN_SHADOW_START)],
+ __pgd(__pa(tmp_pmd_table) | PMD_TYPE_TABLE | L_PGD_SWAPPER));
+#endif
+ cpu_switch_mm(tmp_pgd_table, &init_mm);
+ local_flush_tlb_all();
+
+ clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
+
+ kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_START),
+ kasan_mem_to_shadow((void *)-1UL) + 1);
+
+ for_each_mem_range(i, &pa_start, &pa_end) {
+ void *start = __va(pa_start);
+ void *end = __va(pa_end);
+
+ /* Do not attempt to shadow highmem */
+ if (pa_start >= arm_lowmem_limit) {
+ pr_info("Skip highmem block at %pa-%pa\n", &pa_start, &pa_end);
+ continue;
+ }
+ if (pa_end > arm_lowmem_limit) {
+ pr_info("Truncating shadow for memory block at %pa-%pa to lowmem region at %pa\n",
+ &pa_start, &pa_end, &arm_lowmem_limit);
+ end = __va(arm_lowmem_limit);
+ }
+ if (start >= end) {
+ pr_info("Skipping invalid memory block %pa-%pa (virtual %p-%p)\n",
+ &pa_start, &pa_end, start, end);
+ continue;
+ }
+
+ create_mapping(start, end);
+ }
+
+ /*
+ * 1. The module global variables are in MODULES_VADDR ~ MODULES_END,
+ * so we need to map this area.
+ * 2. PKMAP_BASE ~ PKMAP_BASE+PMD_SIZE's shadow and MODULES_VADDR
+ * ~ MODULES_END's shadow is in the same PMD_SIZE, so we can't
+ * use kasan_populate_zero_shadow.
+ */
+ create_mapping((void *)MODULES_VADDR, (void *)(PKMAP_BASE + PMD_SIZE));
+
+ /*
+ * KAsan may reuse the contents of kasan_early_shadow_pte directly, so
+ * we should make sure that it maps the zero page read-only.
+ */
+ for (i = 0; i < PTRS_PER_PTE; i++)
+ set_pte_at(&init_mm, KASAN_SHADOW_START + i*PAGE_SIZE,
+ &kasan_early_shadow_pte[i],
+ pfn_pte(virt_to_pfn(kasan_early_shadow_page),
+ __pgprot(pgprot_val(PAGE_KERNEL)
+ | L_PTE_RDONLY)));
+
+ cpu_switch_mm(swapper_pg_dir, &init_mm);
+ local_flush_tlb_all();
+
+ memset(kasan_early_shadow_page, 0, PAGE_SIZE);
+ pr_info("Kernel address sanitizer initialized\n");
+ init_task.kasan_depth = 0;
+}
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index ab69250a86bc..c06ebfbc48c4 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -29,6 +29,7 @@
#include <asm/procinfo.h>
#include <asm/memory.h>
#include <asm/pgalloc.h>
+#include <asm/kasan_def.h>
#include <asm/mach/arch.h>
#include <asm/mach/map.h>
@@ -39,6 +40,8 @@
#include "mm.h"
#include "tcm.h"
+extern unsigned long __atags_pointer;
+
/*
* empty_zero_page is a special page that is used for
* zero-initialized data and COW.
@@ -946,7 +949,7 @@ static void __init create_mapping(struct map_desc *md)
return;
}
- if ((md->type == MT_DEVICE || md->type == MT_ROM) &&
+ if (md->type == MT_DEVICE &&
md->virtual >= PAGE_OFFSET && md->virtual < FIXADDR_START &&
(md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) {
pr_warn("BUG: mapping for 0x%08llx at 0x%08lx out of vmalloc space\n",
@@ -1253,8 +1256,25 @@ static inline void prepare_page_table(void)
/*
* Clear out all the mappings below the kernel image.
*/
+#ifdef CONFIG_KASAN
+ /*
+ * KASan's shadow memory inserts itself between the TASK_SIZE
+ * and MODULES_VADDR. Do not clear the KASan shadow memory mappings.
+ */
+ for (addr = 0; addr < KASAN_SHADOW_START; addr += PMD_SIZE)
+ pmd_clear(pmd_off_k(addr));
+ /*
+ * Skip over the KASan shadow area. KASAN_SHADOW_END is sometimes
+ * equal to MODULES_VADDR and then we exit the pmd clearing. If we
+ * are using a thumb-compiled kernel, there there will be 8MB more
+ * to clear as KASan always offset to 16 MB below MODULES_VADDR.
+ */
+ for (addr = KASAN_SHADOW_END; addr < MODULES_VADDR; addr += PMD_SIZE)
+ pmd_clear(pmd_off_k(addr));
+#else
for (addr = 0; addr < MODULES_VADDR; addr += PMD_SIZE)
pmd_clear(pmd_off_k(addr));
+#endif
#ifdef CONFIG_XIP_KERNEL
/* The XIP kernel is mapped in the module area -- skip over it */
@@ -1333,6 +1353,15 @@ static void __init devicemaps_init(const struct machine_desc *mdesc)
for (addr = VMALLOC_START; addr < (FIXADDR_TOP & PMD_MASK); addr += PMD_SIZE)
pmd_clear(pmd_off_k(addr));
+ if (__atags_pointer) {
+ /* create a read-only mapping of the device tree */
+ map.pfn = __phys_to_pfn(__atags_pointer & SECTION_MASK);
+ map.virtual = FDT_FIXED_BASE;
+ map.length = FDT_FIXED_SIZE;
+ map.type = MT_ROM;
+ create_mapping(&map);
+ }
+
/*
* Map the kernel if it is XIP.
* It is always first in the modulearea.
@@ -1489,8 +1518,7 @@ static void __init map_lowmem(void)
}
#ifdef CONFIG_ARM_PV_FIXUP
-extern unsigned long __atags_pointer;
-typedef void pgtables_remap(long long offset, unsigned long pgd, void *bdata);
+typedef void pgtables_remap(long long offset, unsigned long pgd);
pgtables_remap lpae_pgtables_remap_asm;
/*
@@ -1503,7 +1531,6 @@ static void __init early_paging_init(const struct machine_desc *mdesc)
unsigned long pa_pgd;
unsigned int cr, ttbcr;
long long offset;
- void *boot_data;
if (!mdesc->pv_fixup)
return;
@@ -1520,7 +1547,6 @@ static void __init early_paging_init(const struct machine_desc *mdesc)
*/
lpae_pgtables_remap = (pgtables_remap *)(unsigned long)__pa(lpae_pgtables_remap_asm);
pa_pgd = __pa(swapper_pg_dir);
- boot_data = __va(__atags_pointer);
barrier();
pr_info("Switching physical address space to 0x%08llx\n",
@@ -1556,7 +1582,7 @@ static void __init early_paging_init(const struct machine_desc *mdesc)
* needs to be assembly. It's fairly simple, as we're using the
* temporary tables setup by the initial assembly code.
*/
- lpae_pgtables_remap(offset, pa_pgd, boot_data);
+ lpae_pgtables_remap(offset, pa_pgd);
/* Re-enable the caches and cacheable TLB walks */
asm volatile("mcr p15, 0, %0, c2, c0, 2" : : "r" (ttbcr));
diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index c5e1b27046a8..f8e9bc58a84f 100644
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -66,7 +66,21 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
new_pmd = pmd_alloc(mm, new_pud, 0);
if (!new_pmd)
goto no_pmd;
-#endif
+#ifdef CONFIG_KASAN
+ /*
+ * Copy PMD table for KASAN shadow mappings.
+ */
+ init_pgd = pgd_offset_k(TASK_SIZE);
+ init_p4d = p4d_offset(init_pgd, TASK_SIZE);
+ init_pud = pud_offset(init_p4d, TASK_SIZE);
+ init_pmd = pmd_offset(init_pud, TASK_SIZE);
+ new_pmd = pmd_offset(new_pud, TASK_SIZE);
+ memcpy(new_pmd, init_pmd,
+ (pmd_index(MODULES_VADDR) - pmd_index(TASK_SIZE))
+ * sizeof(pmd_t));
+ clean_dcache_area(new_pmd, PTRS_PER_PMD * sizeof(pmd_t));
+#endif /* CONFIG_KASAN */
+#endif /* CONFIG_LPAE */
if (!vectors_high()) {
/*
diff --git a/arch/arm/mm/pv-fixup-asm.S b/arch/arm/mm/pv-fixup-asm.S
index 8eade0416739..5c5e1952000a 100644
--- a/arch/arm/mm/pv-fixup-asm.S
+++ b/arch/arm/mm/pv-fixup-asm.S
@@ -39,8 +39,8 @@ ENTRY(lpae_pgtables_remap_asm)
/* Update level 2 entries for the boot data */
add r7, r2, #0x1000
- add r7, r7, r3, lsr #SECTION_SHIFT - L2_ORDER
- bic r7, r7, #(1 << L2_ORDER) - 1
+ movw r3, #FDT_FIXED_BASE >> (SECTION_SHIFT - L2_ORDER)
+ add r7, r7, r3
ldrd r4, r5, [r7]
adds r4, r4, r0
adc r5, r5, r1
diff --git a/arch/arm/vdso/Makefile b/arch/arm/vdso/Makefile
index 150ce6e6a5d3..b558bee0e1f6 100644
--- a/arch/arm/vdso/Makefile
+++ b/arch/arm/vdso/Makefile
@@ -42,6 +42,8 @@ GCOV_PROFILE := n
# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in.
KCOV_INSTRUMENT := n
+KASAN_SANITIZE := n
+
# Force dependency
$(obj)/vdso.o : $(obj)/vdso.so
diff --git a/arch/arm/vfp/entry.S b/arch/arm/vfp/entry.S
index 0186cf9da890..27b0a1f27fbd 100644
--- a/arch/arm/vfp/entry.S
+++ b/arch/arm/vfp/entry.S
@@ -37,20 +37,3 @@ ENDPROC(vfp_null_entry)
.align 2
.LCvfp:
.word vfp_vector
-
-@ This code is called if the VFP does not exist. It needs to flag the
-@ failure to the VFP initialisation code.
-
- __INIT
-ENTRY(vfp_testing_entry)
- dec_preempt_count_ti r10, r4
- ldr r0, VFP_arch_address
- str r0, [r0] @ set to non-zero value
- ret r9 @ we have handled the fault
-ENDPROC(vfp_testing_entry)
-
- .align 2
-VFP_arch_address:
- .word VFP_arch
-
- __FINIT
diff --git a/arch/arm/vfp/vfphw.S b/arch/arm/vfp/vfphw.S
index 4fcff9f59947..d5837bf05a9a 100644
--- a/arch/arm/vfp/vfphw.S
+++ b/arch/arm/vfp/vfphw.S
@@ -79,11 +79,6 @@ ENTRY(vfp_support_entry)
DBGSTR3 "instr %08x pc %08x state %p", r0, r2, r10
.fpu vfpv2
- ldr r3, [sp, #S_PSR] @ Neither lazy restore nor FP exceptions
- and r3, r3, #MODE_MASK @ are supported in kernel mode
- teq r3, #USR_MODE
- bne vfp_kmode_exception @ Returns through lr
-
VFPFMRX r1, FPEXC @ Is the VFP enabled?
DBGSTR1 "fpexc %08x", r1
tst r1, #FPEXC_EN
diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c
index 8c9e7f9f0277..2cb355c1b5b7 100644
--- a/arch/arm/vfp/vfpmodule.c
+++ b/arch/arm/vfp/vfpmodule.c
@@ -23,6 +23,7 @@
#include <asm/cputype.h>
#include <asm/system_info.h>
#include <asm/thread_notify.h>
+#include <asm/traps.h>
#include <asm/vfp.h>
#include "vfpinstr.h"
@@ -31,7 +32,6 @@
/*
* Our undef handlers (in entry.S)
*/
-asmlinkage void vfp_testing_entry(void);
asmlinkage void vfp_support_entry(void);
asmlinkage void vfp_null_entry(void);
@@ -42,7 +42,7 @@ asmlinkage void (*vfp_vector)(void) = vfp_null_entry;
* Used in startup: set to non-zero if VFP checks fail
* After startup, holds VFP architecture
*/
-unsigned int VFP_arch;
+static unsigned int __initdata VFP_arch;
/*
* The pointer to the vfpstate structure of the thread which currently
@@ -436,7 +436,7 @@ static void vfp_enable(void *unused)
* present on all CPUs within a SMP complex. Needs to be called prior to
* vfp_init().
*/
-void vfp_disable(void)
+void __init vfp_disable(void)
{
if (VFP_arch) {
pr_debug("%s: should be called prior to vfp_init\n", __func__);
@@ -642,7 +642,9 @@ static int vfp_starting_cpu(unsigned int unused)
return 0;
}
-void vfp_kmode_exception(void)
+#ifdef CONFIG_KERNEL_MODE_NEON
+
+static int vfp_kmode_exception(struct pt_regs *regs, unsigned int instr)
{
/*
* If we reach this point, a floating point exception has been raised
@@ -660,9 +662,51 @@ void vfp_kmode_exception(void)
pr_crit("BUG: unsupported FP instruction in kernel mode\n");
else
pr_crit("BUG: FP instruction issued in kernel mode with FP unit disabled\n");
+ pr_crit("FPEXC == 0x%08x\n", fmrx(FPEXC));
+ return 1;
}
-#ifdef CONFIG_KERNEL_MODE_NEON
+static struct undef_hook vfp_kmode_exception_hook[] = {{
+ .instr_mask = 0xfe000000,
+ .instr_val = 0xf2000000,
+ .cpsr_mask = MODE_MASK | PSR_T_BIT,
+ .cpsr_val = SVC_MODE,
+ .fn = vfp_kmode_exception,
+}, {
+ .instr_mask = 0xff100000,
+ .instr_val = 0xf4000000,
+ .cpsr_mask = MODE_MASK | PSR_T_BIT,
+ .cpsr_val = SVC_MODE,
+ .fn = vfp_kmode_exception,
+}, {
+ .instr_mask = 0xef000000,
+ .instr_val = 0xef000000,
+ .cpsr_mask = MODE_MASK | PSR_T_BIT,
+ .cpsr_val = SVC_MODE | PSR_T_BIT,
+ .fn = vfp_kmode_exception,
+}, {
+ .instr_mask = 0xff100000,
+ .instr_val = 0xf9000000,
+ .cpsr_mask = MODE_MASK | PSR_T_BIT,
+ .cpsr_val = SVC_MODE | PSR_T_BIT,
+ .fn = vfp_kmode_exception,
+}, {
+ .instr_mask = 0x0c000e00,
+ .instr_val = 0x0c000a00,
+ .cpsr_mask = MODE_MASK,
+ .cpsr_val = SVC_MODE,
+ .fn = vfp_kmode_exception,
+}};
+
+static int __init vfp_kmode_exception_hook_init(void)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(vfp_kmode_exception_hook); i++)
+ register_undef_hook(&vfp_kmode_exception_hook[i]);
+ return 0;
+}
+subsys_initcall(vfp_kmode_exception_hook_init);
/*
* Kernel-side NEON support functions
@@ -708,6 +752,21 @@ EXPORT_SYMBOL(kernel_neon_end);
#endif /* CONFIG_KERNEL_MODE_NEON */
+static int __init vfp_detect(struct pt_regs *regs, unsigned int instr)
+{
+ VFP_arch = UINT_MAX; /* mark as not present */
+ regs->ARM_pc += 4;
+ return 0;
+}
+
+static struct undef_hook vfp_detect_hook __initdata = {
+ .instr_mask = 0x0c000e00,
+ .instr_val = 0x0c000a00,
+ .cpsr_mask = MODE_MASK,
+ .cpsr_val = SVC_MODE,
+ .fn = vfp_detect,
+};
+
/*
* VFP support code initialisation.
*/
@@ -728,10 +787,11 @@ static int __init vfp_init(void)
* The handler is already setup to just log calls, so
* we just need to read the VFPSID register.
*/
- vfp_vector = vfp_testing_entry;
+ register_undef_hook(&vfp_detect_hook);
barrier();
vfpsid = fmrx(FPSID);
barrier();
+ unregister_undef_hook(&vfp_detect_hook);
vfp_vector = vfp_null_entry;
pr_info("VFP support v0.3: ");