diff options
| author | Ard Biesheuvel <ardb@kernel.org> | 2021-09-18 10:44:37 +0200 |
|---|---|---|
| committer | Ard Biesheuvel <ardb@kernel.org> | 2021-09-27 16:54:02 +0200 |
| commit | 50596b7559bf226bb35ad55855ee979453ec06a1 (patch) | |
| tree | 844d4e625c603c1c76ebfd1e21abe9e657e531e0 /arch/arm/kernel | |
| parent | 3855ab614df4818c833864572559a97fd9f9a299 (diff) | |
ARM: smp: Store current pointer in TPIDRURO register if available
Now that the user space TLS register is assigned on every return to user
space, we can use it to keep the 'current' pointer while running in the
kernel. This removes the need to access it via thread_info, which is
located at the base of the stack, but will be moved out of there in a
subsequent patch.
Use the __builtin_thread_pointer() helper when available - this will
help GCC understand that reloading the value within the same function is
not necessary, even when using the per-task stack protector (which also
generates accesses via the TLS register). For example, the generated
code below loads TPIDRURO only once, and uses it to access both the
stack canary and the preempt_count fields.
<do_one_initcall>:
e92d 41f0 stmdb sp!, {r4, r5, r6, r7, r8, lr}
ee1d 4f70 mrc 15, 0, r4, cr13, cr0, {3}
4606 mov r6, r0
b094 sub sp, #80 ; 0x50
f8d4 34e8 ldr.w r3, [r4, #1256] ; 0x4e8 <- stack canary
9313 str r3, [sp, #76] ; 0x4c
f8d4 8004 ldr.w r8, [r4, #4] <- preempt count
Co-developed-by: Keith Packard <keithpac@amazon.com>
Signed-off-by: Keith Packard <keithpac@amazon.com>
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Linus Walleij <linus.walleij@linaro.org>
Tested-by: Amit Daniel Kachhap <amit.kachhap@arm.com>
Diffstat (limited to 'arch/arm/kernel')
| -rw-r--r-- | arch/arm/kernel/entry-armv.S | 5 | ||||
| -rw-r--r-- | arch/arm/kernel/entry-common.S | 1 | ||||
| -rw-r--r-- | arch/arm/kernel/head-common.S | 5 | ||||
| -rw-r--r-- | arch/arm/kernel/process.c | 4 | ||||
| -rw-r--r-- | arch/arm/kernel/smp.c | 2 |
5 files changed, 17 insertions, 0 deletions
diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 241b73d64df7..7263a45abf3d 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -384,6 +384,8 @@ ENDPROC(__fiq_abt) ATRAP( teq r8, r7) ATRAP( mcrne p15, 0, r8, c1, c0, 0) + reload_current r7, r8 + @ @ Clear FP to mark the first stack frame @ @@ -762,6 +764,8 @@ ENTRY(__switch_to) add r7, r7, #TSK_STACK_CANARY & ~IMM12_MASK .endif ldr r7, [r7, #TSK_STACK_CANARY & IMM12_MASK] +#elif defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) + ldr r7, [r2, #TI_TASK] #endif #ifdef CONFIG_CPU_USE_DOMAINS mcr p15, 0, r6, c3, c0, 0 @ Set domain register @@ -776,6 +780,7 @@ ENTRY(__switch_to) #endif THUMB( mov ip, r4 ) mov r0, r5 + set_current r7 ARM( ldmia r4, {r4 - sl, fp, sp, pc} ) @ Load all regs saved previously THUMB( ldmia ip!, {r4 - sl, fp} ) @ Load all regs saved previously THUMB( ldr sp, [ip], #4 ) diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index d9c99db50243..ac86c34682bb 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -170,6 +170,7 @@ ENTRY(vector_swi) str saved_psr, [sp, #S_PSR] @ Save CPSR str r0, [sp, #S_OLD_R0] @ Save OLD_R0 #endif + reload_current r10, ip zero_fp alignment_trap r10, ip, __cr_alignment asm_trace_hardirqs_on save=0 diff --git a/arch/arm/kernel/head-common.S b/arch/arm/kernel/head-common.S index 29b2eda136bb..da18e0a17dc2 100644 --- a/arch/arm/kernel/head-common.S +++ b/arch/arm/kernel/head-common.S @@ -105,6 +105,11 @@ __mmap_switched: mov r1, #0 bl __memset @ clear .bss +#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO + adr_l r0, init_task @ get swapper task_struct + set_current r0 +#endif + ldmia r4, {r0, r1, r2, r3} str r9, [r0] @ Save processor ID str r7, [r1] @ Save machine type diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index cd73c216b272..30428d756515 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -36,6 +36,10 @@ #include "signal.h" +#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO +DEFINE_PER_CPU(struct task_struct *, __entry_task); +#endif + #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) #include <linux/stackprotector.h> unsigned long __stack_chk_guard __read_mostly; diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 8979d548ec17..97ee6b1567e9 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -409,6 +409,8 @@ asmlinkage void secondary_start_kernel(struct task_struct *task) struct mm_struct *mm = &init_mm; unsigned int cpu; + set_current(task); + secondary_biglittle_init(); /* |