summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm/Kconfig5
-rw-r--r--arch/arm/Makefile4
-rw-r--r--arch/arm/include/asm/assembler.h24
-rw-r--r--arch/arm/include/asm/current.h50
-rw-r--r--arch/arm/include/asm/switch_to.h2
-rw-r--r--arch/arm/include/asm/thread_info.h2
-rw-r--r--arch/arm/kernel/entry-armv.S5
-rw-r--r--arch/arm/kernel/entry-common.S1
-rw-r--r--arch/arm/kernel/head-common.S5
-rw-r--r--arch/arm/kernel/process.c4
-rw-r--r--arch/arm/kernel/smp.c2
-rw-r--r--arch/arm/mm/proc-macros.S3
12 files changed, 105 insertions, 2 deletions
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index ff3e64ae959e..cd195e6f4ea6 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -1157,6 +1157,11 @@ config SMP_ON_UP
If you don't know what to do here, say Y.
+
+config CURRENT_POINTER_IN_TPIDRURO
+ def_bool y
+ depends on SMP && CPU_32v6K && !CPU_V6
+
config ARM_CPU_TOPOLOGY
bool "Support cpu topology definition"
depends on SMP && CPU_V7
diff --git a/arch/arm/Makefile b/arch/arm/Makefile
index b46e673a0ebe..1c540157e283 100644
--- a/arch/arm/Makefile
+++ b/arch/arm/Makefile
@@ -113,6 +113,10 @@ ifeq ($(CONFIG_CC_IS_CLANG),y)
CFLAGS_ABI += -meabi gnu
endif
+ifeq ($(CONFIG_CURRENT_POINTER_IN_TPIDRURO),y)
+CFLAGS_ABI += -mtp=cp15
+endif
+
# Accept old syntax despite ".syntax unified"
AFLAGS_NOWARN :=$(call as-option,-Wa$(comma)-mno-warn-deprecated,-Wa$(comma)-W)
diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h
index e2b1fd558bf3..c1551dee28be 100644
--- a/arch/arm/include/asm/assembler.h
+++ b/arch/arm/include/asm/assembler.h
@@ -199,6 +199,30 @@
.endm
.endr
+ .macro get_current, rd
+#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
+ mrc p15, 0, \rd, c13, c0, 3 @ get TPIDRURO register
+#else
+ get_thread_info \rd
+ ldr \rd, [\rd, #TI_TASK]
+#endif
+ .endm
+
+ .macro set_current, rn
+#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
+ mcr p15, 0, \rn, c13, c0, 3 @ set TPIDRURO register
+#endif
+ .endm
+
+ .macro reload_current, t1:req, t2:req
+#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
+ adr_l \t1, __entry_task @ get __entry_task base address
+ mrc p15, 0, \t2, c13, c0, 4 @ get per-CPU offset
+ ldr \t1, [\t1, \t2] @ load variable
+ mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO
+#endif
+ .endm
+
/*
* Get current thread_info.
*/
diff --git a/arch/arm/include/asm/current.h b/arch/arm/include/asm/current.h
new file mode 100644
index 000000000000..1d472fa7697b
--- /dev/null
+++ b/arch/arm/include/asm/current.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2021 Keith Packard <keithp@keithp.com>
+ * Copyright (c) 2021 Google, LLC <ardb@kernel.org>
+ */
+
+#ifndef _ASM_ARM_CURRENT_H
+#define _ASM_ARM_CURRENT_H
+
+#ifndef __ASSEMBLY__
+
+struct task_struct;
+
+static inline void set_current(struct task_struct *cur)
+{
+ if (!IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO))
+ return;
+
+ /* Set TPIDRURO */
+ asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory");
+}
+
+#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
+
+static inline struct task_struct *get_current(void)
+{
+ struct task_struct *cur;
+
+#if __has_builtin(__builtin_thread_pointer)
+ /*
+ * Use the __builtin helper when available - this results in better
+ * code, especially when using GCC in combination with the per-task
+ * stack protector, as the compiler will recognize that it needs to
+ * load the TLS register only once in every function.
+ */
+ cur = __builtin_thread_pointer();
+#else
+ asm("mrc p15, 0, %0, c13, c0, 3" : "=r"(cur));
+#endif
+ return cur;
+}
+
+#define current get_current()
+#else
+#include <asm-generic/current.h>
+#endif /* CONFIG_CURRENT_POINTER_IN_TPIDRURO */
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* _ASM_ARM_CURRENT_H */
diff --git a/arch/arm/include/asm/switch_to.h b/arch/arm/include/asm/switch_to.h
index 007d8fea7157..61e4a3c4ca6e 100644
--- a/arch/arm/include/asm/switch_to.h
+++ b/arch/arm/include/asm/switch_to.h
@@ -26,6 +26,8 @@ extern struct task_struct *__switch_to(struct task_struct *, struct thread_info
#define switch_to(prev,next,last) \
do { \
__complete_pending_tlbi(); \
+ if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO)) \
+ __this_cpu_write(__entry_task, next); \
last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \
} while (0)
diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h
index f0cacc733231..76b6fbd5540c 100644
--- a/arch/arm/include/asm/thread_info.h
+++ b/arch/arm/include/asm/thread_info.h
@@ -29,6 +29,8 @@
struct task_struct;
+DECLARE_PER_CPU(struct task_struct *, __entry_task);
+
#include <asm/types.h>
struct cpu_context_save {
diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S
index 241b73d64df7..7263a45abf3d 100644
--- a/arch/arm/kernel/entry-armv.S
+++ b/arch/arm/kernel/entry-armv.S
@@ -384,6 +384,8 @@ ENDPROC(__fiq_abt)
ATRAP( teq r8, r7)
ATRAP( mcrne p15, 0, r8, c1, c0, 0)
+ reload_current r7, r8
+
@
@ Clear FP to mark the first stack frame
@
@@ -762,6 +764,8 @@ ENTRY(__switch_to)
add r7, r7, #TSK_STACK_CANARY & ~IMM12_MASK
.endif
ldr r7, [r7, #TSK_STACK_CANARY & IMM12_MASK]
+#elif defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO)
+ ldr r7, [r2, #TI_TASK]
#endif
#ifdef CONFIG_CPU_USE_DOMAINS
mcr p15, 0, r6, c3, c0, 0 @ Set domain register
@@ -776,6 +780,7 @@ ENTRY(__switch_to)
#endif
THUMB( mov ip, r4 )
mov r0, r5
+ set_current r7
ARM( ldmia r4, {r4 - sl, fp, sp, pc} ) @ Load all regs saved previously
THUMB( ldmia ip!, {r4 - sl, fp} ) @ Load all regs saved previously
THUMB( ldr sp, [ip], #4 )
diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S
index d9c99db50243..ac86c34682bb 100644
--- a/arch/arm/kernel/entry-common.S
+++ b/arch/arm/kernel/entry-common.S
@@ -170,6 +170,7 @@ ENTRY(vector_swi)
str saved_psr, [sp, #S_PSR] @ Save CPSR
str r0, [sp, #S_OLD_R0] @ Save OLD_R0
#endif
+ reload_current r10, ip
zero_fp
alignment_trap r10, ip, __cr_alignment
asm_trace_hardirqs_on save=0
diff --git a/arch/arm/kernel/head-common.S b/arch/arm/kernel/head-common.S
index 29b2eda136bb..da18e0a17dc2 100644
--- a/arch/arm/kernel/head-common.S
+++ b/arch/arm/kernel/head-common.S
@@ -105,6 +105,11 @@ __mmap_switched:
mov r1, #0
bl __memset @ clear .bss
+#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
+ adr_l r0, init_task @ get swapper task_struct
+ set_current r0
+#endif
+
ldmia r4, {r0, r1, r2, r3}
str r9, [r0] @ Save processor ID
str r7, [r1] @ Save machine type
diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c
index cd73c216b272..30428d756515 100644
--- a/arch/arm/kernel/process.c
+++ b/arch/arm/kernel/process.c
@@ -36,6 +36,10 @@
#include "signal.h"
+#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
+DEFINE_PER_CPU(struct task_struct *, __entry_task);
+#endif
+
#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
#include <linux/stackprotector.h>
unsigned long __stack_chk_guard __read_mostly;
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index 8979d548ec17..97ee6b1567e9 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -409,6 +409,8 @@ asmlinkage void secondary_start_kernel(struct task_struct *task)
struct mm_struct *mm = &init_mm;
unsigned int cpu;
+ set_current(task);
+
secondary_biglittle_init();
/*
diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S
index e2c743aa2eb2..d48ba99d739c 100644
--- a/arch/arm/mm/proc-macros.S
+++ b/arch/arm/mm/proc-macros.S
@@ -30,8 +30,7 @@
* act_mm - get current->active_mm
*/
.macro act_mm, rd
- get_thread_info \rd
- ldr \rd, [\rd, #TI_TASK]
+ get_current \rd
.if (TSK_ACTIVE_MM > IMM12_MASK)
add \rd, \rd, #TSK_ACTIVE_MM & ~IMM12_MASK
.endif
generated by cgit (git 2.34.1) at 2024-05-28 16:48:31 +0000