From 7b9896c352073156a325c3bb0dc4c46e06e2a468 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 25 Nov 2021 10:26:44 +0100 Subject: ARM: percpu: add SMP_ON_UP support Permit the use of the TPIDRPRW system register for carrying the per-CPU offset in generic SMP configurations that also target non-SMP capable ARMv6 cores. This uses the SMP_ON_UP code patching framework to turn all TPIDRPRW accesses into reads/writes of entry #0 in the __per_cpu_offset array. While at it, switch over some existing direct TPIDRPRW accesses in asm code to invocations of a new helper that is patched in the same way when necessary. Note that CPU_V6+SMP without SMP_ON_UP results in a kernel that does not boot on v6 CPUs without SMP extensions, so add this dependency to Kconfig as well. Acked-by: Linus Walleij Acked-by: Nicolas Pitre Signed-off-by: Ard Biesheuvel Tested-by: Marc Zyngier Tested-by: Vladimir Murzin # ARMv7M --- arch/arm/include/asm/assembler.h | 59 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) (limited to 'arch/arm/include/asm/assembler.h') diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 2095638b7140..f9b3dd0e9ef5 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -220,9 +220,7 @@ THUMB( fpreg .req r7 ) .macro reload_current, t1:req, t2:req #ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - adr_l \t1, __entry_task @ get __entry_task base address - mrc p15, 0, \t2, c13, c0, 4 @ get per-CPU offset - ldr \t1, [\t1, \t2] @ load variable + ldr_this_cpu \t1, __entry_task, \t1, \t2 mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO #endif .endm @@ -312,6 +310,26 @@ THUMB( fpreg .req r7 ) #define ALT_UP_B(label) b label #endif + /* + * this_cpu_offset - load the per-CPU offset of this CPU into + * register 'rd' + */ + .macro this_cpu_offset, rd:req +#ifdef CONFIG_SMP +ALT_SMP(mrc p15, 0, \rd, c13, c0, 4) +#ifdef CONFIG_CPU_V6 +ALT_UP_B(.L1_\@) +.L0_\@: + .subsection 1 +.L1_\@: ldr_va \rd, __per_cpu_offset + b .L0_\@ + .previous +#endif +#else + mov \rd, #0 +#endif + .endm + /* * Instruction barrier */ @@ -648,6 +666,41 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) __ldst_va str, \rn, \tmp, \sym, \cond .endm + /* + * ldr_this_cpu_armv6 - Load a 32-bit word from the per-CPU variable 'sym', + * without using a temp register. Supported in ARM mode + * only. + */ + .macro ldr_this_cpu_armv6, rd:req, sym:req + this_cpu_offset \rd + .globl \sym + .reloc .L0_\@, R_ARM_ALU_PC_G0_NC, \sym + .reloc .L1_\@, R_ARM_ALU_PC_G1_NC, \sym + .reloc .L2_\@, R_ARM_LDR_PC_G2, \sym + add \rd, \rd, pc +.L0_\@: sub \rd, \rd, #4 +.L1_\@: sub \rd, \rd, #0 +.L2_\@: ldr \rd, [\rd, #4] + .endm + + /* + * ldr_this_cpu - Load a 32-bit word from the per-CPU variable 'sym' + * into register 'rd', which may be the stack pointer, + * using 't1' and 't2' as general temp registers. These + * are permitted to overlap with 'rd' if != sp + */ + .macro ldr_this_cpu, rd:req, sym:req, t1:req, t2:req +#if __LINUX_ARM_ARCH__ >= 7 || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \ + (defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000) + this_cpu_offset \t1 + mov_l \t2, \sym + ldr \rd, [\t1, \t2] +#else + ldr_this_cpu_armv6 \rd, \sym +#endif + .endm + /* * rev_l - byte-swap a 32-bit value * -- cgit