diff options
Diffstat (limited to 'arch/loongarch/include')
32 files changed, 980 insertions, 335 deletions
diff --git a/arch/loongarch/include/asm/Kbuild b/arch/loongarch/include/asm/Kbuild index b04d2cef935f..9034b583a88a 100644 --- a/arch/loongarch/include/asm/Kbuild +++ b/arch/loongarch/include/asm/Kbuild @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 +syscall-y += syscall_table_32.h syscall-y += syscall_table_64.h generated-y += orc_hash.h diff --git a/arch/loongarch/include/asm/addrspace.h b/arch/loongarch/include/asm/addrspace.h index e739dbc6329d..d6472cafb32c 100644 --- a/arch/loongarch/include/asm/addrspace.h +++ b/arch/loongarch/include/asm/addrspace.h @@ -38,11 +38,20 @@ extern unsigned long vm_map_base; #endif #ifndef WRITECOMBINE_BASE +#ifdef CONFIG_32BIT +#define WRITECOMBINE_BASE CSR_DMW0_BASE +#else #define WRITECOMBINE_BASE CSR_DMW2_BASE #endif +#endif +#ifdef CONFIG_32BIT +#define DMW_PABITS 29 +#define TO_PHYS_MASK ((_UL(1) << _UL(DMW_PABITS)) - 1) +#else #define DMW_PABITS 48 -#define TO_PHYS_MASK ((1ULL << DMW_PABITS) - 1) +#define TO_PHYS_MASK ((_ULL(1) << _ULL(DMW_PABITS)) - 1) +#endif /* * Memory above this physical address will be considered highmem. @@ -112,7 +121,11 @@ extern unsigned long vm_map_base; /* * Returns the physical address of a KPRANGEx / XKPRANGE address */ +#ifdef CONFIG_32BIT +#define PHYSADDR(a) ((_ACAST32_(a)) & TO_PHYS_MASK) +#else #define PHYSADDR(a) ((_ACAST64_(a)) & TO_PHYS_MASK) +#endif /* * On LoongArch, I/O ports mappring is following: diff --git a/arch/loongarch/include/asm/asm.h b/arch/loongarch/include/asm/asm.h index f018d26fc995..719cab1a0ad8 100644 --- a/arch/loongarch/include/asm/asm.h +++ b/arch/loongarch/include/asm/asm.h @@ -72,11 +72,11 @@ #define INT_SUB sub.w #define INT_L ld.w #define INT_S st.w -#define INT_SLL slli.w +#define INT_SLLI slli.w #define INT_SLLV sll.w -#define INT_SRL srli.w +#define INT_SRLI srli.w #define INT_SRLV srl.w -#define INT_SRA srai.w +#define INT_SRAI srai.w #define INT_SRAV sra.w #endif @@ -86,11 +86,11 @@ #define INT_SUB sub.d #define INT_L ld.d #define INT_S st.d -#define INT_SLL slli.d +#define INT_SLLI slli.d #define INT_SLLV sll.d -#define INT_SRL srli.d +#define INT_SRLI srli.d #define INT_SRLV srl.d -#define INT_SRA srai.d +#define INT_SRAI srai.d #define INT_SRAV sra.d #endif @@ -100,15 +100,23 @@ #if (__SIZEOF_LONG__ == 4) #define LONG_ADD add.w #define LONG_ADDI addi.w +#define LONG_ALSL alsl.w +#define LONG_BSTRINS bstrins.w +#define LONG_BSTRPICK bstrpick.w #define LONG_SUB sub.w #define LONG_L ld.w +#define LONG_LI li.w +#define LONG_LPTR ld.w #define LONG_S st.w -#define LONG_SLL slli.w +#define LONG_SPTR st.w +#define LONG_SLLI slli.w #define LONG_SLLV sll.w -#define LONG_SRL srli.w +#define LONG_SRLI srli.w #define LONG_SRLV srl.w -#define LONG_SRA srai.w +#define LONG_SRAI srai.w #define LONG_SRAV sra.w +#define LONG_ROTR rotr.w +#define LONG_ROTRI rotri.w #ifdef __ASSEMBLER__ #define LONG .word @@ -121,15 +129,23 @@ #if (__SIZEOF_LONG__ == 8) #define LONG_ADD add.d #define LONG_ADDI addi.d +#define LONG_ALSL alsl.d +#define LONG_BSTRINS bstrins.d +#define LONG_BSTRPICK bstrpick.d #define LONG_SUB sub.d #define LONG_L ld.d +#define LONG_LI li.d +#define LONG_LPTR ldptr.d #define LONG_S st.d -#define LONG_SLL slli.d +#define LONG_SPTR stptr.d +#define LONG_SLLI slli.d #define LONG_SLLV sll.d -#define LONG_SRL srli.d +#define LONG_SRLI srli.d #define LONG_SRLV srl.d -#define LONG_SRA srai.d +#define LONG_SRAI srai.d #define LONG_SRAV sra.d +#define LONG_ROTR rotr.d +#define LONG_ROTRI rotri.d #ifdef __ASSEMBLER__ #define LONG .dword @@ -145,16 +161,23 @@ #if (__SIZEOF_POINTER__ == 4) #define PTR_ADD add.w #define PTR_ADDI addi.w +#define PTR_ALSL alsl.w +#define PTR_BSTRINS bstrins.w +#define PTR_BSTRPICK bstrpick.w #define PTR_SUB sub.w #define PTR_L ld.w -#define PTR_S st.w #define PTR_LI li.w -#define PTR_SLL slli.w +#define PTR_LPTR ld.w +#define PTR_S st.w +#define PTR_SPTR st.w +#define PTR_SLLI slli.w #define PTR_SLLV sll.w -#define PTR_SRL srli.w +#define PTR_SRLI srli.w #define PTR_SRLV srl.w -#define PTR_SRA srai.w +#define PTR_SRAI srai.w #define PTR_SRAV sra.w +#define PTR_ROTR rotr.w +#define PTR_ROTRI rotri.w #define PTR_SCALESHIFT 2 @@ -168,16 +191,23 @@ #if (__SIZEOF_POINTER__ == 8) #define PTR_ADD add.d #define PTR_ADDI addi.d +#define PTR_ALSL alsl.d +#define PTR_BSTRINS bstrins.d +#define PTR_BSTRPICK bstrpick.d #define PTR_SUB sub.d #define PTR_L ld.d -#define PTR_S st.d #define PTR_LI li.d -#define PTR_SLL slli.d +#define PTR_LPTR ldptr.d +#define PTR_S st.d +#define PTR_SPTR stptr.d +#define PTR_SLLI slli.d #define PTR_SLLV sll.d -#define PTR_SRL srli.d +#define PTR_SRLI srli.d #define PTR_SRLV srl.d -#define PTR_SRA srai.d +#define PTR_SRAI srai.d #define PTR_SRAV sra.d +#define PTR_ROTR rotr.d +#define PTR_ROTRI rotri.d #define PTR_SCALESHIFT 3 @@ -190,10 +220,17 @@ /* Annotate a function as being unsuitable for kprobes. */ #ifdef CONFIG_KPROBES +#ifdef CONFIG_32BIT +#define _ASM_NOKPROBE(name) \ + .pushsection "_kprobe_blacklist", "aw"; \ + .long name; \ + .popsection +#else #define _ASM_NOKPROBE(name) \ .pushsection "_kprobe_blacklist", "aw"; \ .quad name; \ .popsection +#endif #else #define _ASM_NOKPROBE(name) #endif diff --git a/arch/loongarch/include/asm/asmmacro.h b/arch/loongarch/include/asm/asmmacro.h index 8d7f501b0a12..a648be5f723f 100644 --- a/arch/loongarch/include/asm/asmmacro.h +++ b/arch/loongarch/include/asm/asmmacro.h @@ -5,43 +5,55 @@ #ifndef _ASM_ASMMACRO_H #define _ASM_ASMMACRO_H +#include <linux/sizes.h> #include <asm/asm-offsets.h> #include <asm/regdef.h> #include <asm/fpregdef.h> #include <asm/loongarch.h> +#ifdef CONFIG_64BIT +#define TASK_STRUCT_OFFSET 0 +#else +#define TASK_STRUCT_OFFSET 2000 +#endif + .macro cpu_save_nonscratch thread - stptr.d s0, \thread, THREAD_REG23 - stptr.d s1, \thread, THREAD_REG24 - stptr.d s2, \thread, THREAD_REG25 - stptr.d s3, \thread, THREAD_REG26 - stptr.d s4, \thread, THREAD_REG27 - stptr.d s5, \thread, THREAD_REG28 - stptr.d s6, \thread, THREAD_REG29 - stptr.d s7, \thread, THREAD_REG30 - stptr.d s8, \thread, THREAD_REG31 - stptr.d sp, \thread, THREAD_REG03 - stptr.d fp, \thread, THREAD_REG22 + LONG_SPTR s0, \thread, (THREAD_REG23 - TASK_STRUCT_OFFSET) + LONG_SPTR s1, \thread, (THREAD_REG24 - TASK_STRUCT_OFFSET) + LONG_SPTR s2, \thread, (THREAD_REG25 - TASK_STRUCT_OFFSET) + LONG_SPTR s3, \thread, (THREAD_REG26 - TASK_STRUCT_OFFSET) + LONG_SPTR s4, \thread, (THREAD_REG27 - TASK_STRUCT_OFFSET) + LONG_SPTR s5, \thread, (THREAD_REG28 - TASK_STRUCT_OFFSET) + LONG_SPTR s6, \thread, (THREAD_REG29 - TASK_STRUCT_OFFSET) + LONG_SPTR s7, \thread, (THREAD_REG30 - TASK_STRUCT_OFFSET) + LONG_SPTR s8, \thread, (THREAD_REG31 - TASK_STRUCT_OFFSET) + LONG_SPTR ra, \thread, (THREAD_REG01 - TASK_STRUCT_OFFSET) + LONG_SPTR sp, \thread, (THREAD_REG03 - TASK_STRUCT_OFFSET) + LONG_SPTR fp, \thread, (THREAD_REG22 - TASK_STRUCT_OFFSET) .endm .macro cpu_restore_nonscratch thread - ldptr.d s0, \thread, THREAD_REG23 - ldptr.d s1, \thread, THREAD_REG24 - ldptr.d s2, \thread, THREAD_REG25 - ldptr.d s3, \thread, THREAD_REG26 - ldptr.d s4, \thread, THREAD_REG27 - ldptr.d s5, \thread, THREAD_REG28 - ldptr.d s6, \thread, THREAD_REG29 - ldptr.d s7, \thread, THREAD_REG30 - ldptr.d s8, \thread, THREAD_REG31 - ldptr.d ra, \thread, THREAD_REG01 - ldptr.d sp, \thread, THREAD_REG03 - ldptr.d fp, \thread, THREAD_REG22 + LONG_LPTR s0, \thread, (THREAD_REG23 - TASK_STRUCT_OFFSET) + LONG_LPTR s1, \thread, (THREAD_REG24 - TASK_STRUCT_OFFSET) + LONG_LPTR s2, \thread, (THREAD_REG25 - TASK_STRUCT_OFFSET) + LONG_LPTR s3, \thread, (THREAD_REG26 - TASK_STRUCT_OFFSET) + LONG_LPTR s4, \thread, (THREAD_REG27 - TASK_STRUCT_OFFSET) + LONG_LPTR s5, \thread, (THREAD_REG28 - TASK_STRUCT_OFFSET) + LONG_LPTR s6, \thread, (THREAD_REG29 - TASK_STRUCT_OFFSET) + LONG_LPTR s7, \thread, (THREAD_REG30 - TASK_STRUCT_OFFSET) + LONG_LPTR s8, \thread, (THREAD_REG31 - TASK_STRUCT_OFFSET) + LONG_LPTR ra, \thread, (THREAD_REG01 - TASK_STRUCT_OFFSET) + LONG_LPTR sp, \thread, (THREAD_REG03 - TASK_STRUCT_OFFSET) + LONG_LPTR fp, \thread, (THREAD_REG22 - TASK_STRUCT_OFFSET) .endm .macro fpu_save_csr thread tmp movfcsr2gr \tmp, fcsr0 +#ifdef CONFIG_32BIT + st.w \tmp, \thread, THREAD_FCSR +#else stptr.w \tmp, \thread, THREAD_FCSR +#endif #ifdef CONFIG_CPU_HAS_LBT /* TM bit is always 0 if LBT not supported */ andi \tmp, \tmp, FPU_CSR_TM @@ -56,7 +68,11 @@ .endm .macro fpu_restore_csr thread tmp0 tmp1 +#ifdef CONFIG_32BIT + ld.w \tmp0, \thread, THREAD_FCSR +#else ldptr.w \tmp0, \thread, THREAD_FCSR +#endif movgr2fcsr fcsr0, \tmp0 #ifdef CONFIG_CPU_HAS_LBT /* TM bit is always 0 if LBT not supported */ @@ -88,9 +104,52 @@ #endif .endm +#ifdef CONFIG_32BIT .macro fpu_save_cc thread tmp0 tmp1 movcf2gr \tmp0, $fcc0 - move \tmp1, \tmp0 + move \tmp1, \tmp0 + movcf2gr \tmp0, $fcc1 + bstrins.w \tmp1, \tmp0, 15, 8 + movcf2gr \tmp0, $fcc2 + bstrins.w \tmp1, \tmp0, 23, 16 + movcf2gr \tmp0, $fcc3 + bstrins.w \tmp1, \tmp0, 31, 24 + st.w \tmp1, \thread, THREAD_FCC + movcf2gr \tmp0, $fcc4 + move \tmp1, \tmp0 + movcf2gr \tmp0, $fcc5 + bstrins.w \tmp1, \tmp0, 15, 8 + movcf2gr \tmp0, $fcc6 + bstrins.w \tmp1, \tmp0, 23, 16 + movcf2gr \tmp0, $fcc7 + bstrins.w \tmp1, \tmp0, 31, 24 + st.w \tmp1, \thread, (THREAD_FCC + 4) + .endm + + .macro fpu_restore_cc thread tmp0 tmp1 + ld.w \tmp0, \thread, THREAD_FCC + bstrpick.w \tmp1, \tmp0, 7, 0 + movgr2cf $fcc0, \tmp1 + bstrpick.w \tmp1, \tmp0, 15, 8 + movgr2cf $fcc1, \tmp1 + bstrpick.w \tmp1, \tmp0, 23, 16 + movgr2cf $fcc2, \tmp1 + bstrpick.w \tmp1, \tmp0, 31, 24 + movgr2cf $fcc3, \tmp1 + ld.w \tmp0, \thread, (THREAD_FCC + 4) + bstrpick.w \tmp1, \tmp0, 7, 0 + movgr2cf $fcc4, \tmp1 + bstrpick.w \tmp1, \tmp0, 15, 8 + movgr2cf $fcc5, \tmp1 + bstrpick.w \tmp1, \tmp0, 23, 16 + movgr2cf $fcc6, \tmp1 + bstrpick.w \tmp1, \tmp0, 31, 24 + movgr2cf $fcc7, \tmp1 + .endm +#else + .macro fpu_save_cc thread tmp0 tmp1 + movcf2gr \tmp0, $fcc0 + move \tmp1, \tmp0 movcf2gr \tmp0, $fcc1 bstrins.d \tmp1, \tmp0, 15, 8 movcf2gr \tmp0, $fcc2 @@ -109,7 +168,7 @@ .endm .macro fpu_restore_cc thread tmp0 tmp1 - ldptr.d \tmp0, \thread, THREAD_FCC + ldptr.d \tmp0, \thread, THREAD_FCC bstrpick.d \tmp1, \tmp0, 7, 0 movgr2cf $fcc0, \tmp1 bstrpick.d \tmp1, \tmp0, 15, 8 @@ -127,6 +186,7 @@ bstrpick.d \tmp1, \tmp0, 63, 56 movgr2cf $fcc7, \tmp1 .endm +#endif .macro fpu_save_double thread tmp li.w \tmp, THREAD_FPR0 @@ -606,12 +666,14 @@ 766: lu12i.w \reg, 0 ori \reg, \reg, 0 +#ifdef CONFIG_64BIT lu32i.d \reg, 0 lu52i.d \reg, \reg, 0 +#endif .pushsection ".la_abs", "aw", %progbits - .p2align 3 - .dword 766b - .dword \sym + .p2align PTRLOG + PTR 766b + PTR \sym .popsection #endif .endm diff --git a/arch/loongarch/include/asm/atomic-amo.h b/arch/loongarch/include/asm/atomic-amo.h new file mode 100644 index 000000000000..d5efa5252d56 --- /dev/null +++ b/arch/loongarch/include/asm/atomic-amo.h @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Atomic operations (AMO). + * + * Copyright (C) 2020-2025 Loongson Technology Corporation Limited + */ + +#ifndef _ASM_ATOMIC_AMO_H +#define _ASM_ATOMIC_AMO_H + +#include <linux/types.h> +#include <asm/barrier.h> +#include <asm/cmpxchg.h> + +#define ATOMIC_OP(op, I, asm_op) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ +{ \ + __asm__ __volatile__( \ + "am"#asm_op".w" " $zero, %1, %0 \n" \ + : "+ZB" (v->counter) \ + : "r" (I) \ + : "memory"); \ +} + +#define ATOMIC_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \ +static inline int arch_atomic_##op##_return##suffix(int i, atomic_t *v) \ +{ \ + int result; \ + \ + __asm__ __volatile__( \ + "am"#asm_op#mb".w" " %1, %2, %0 \n" \ + : "+ZB" (v->counter), "=&r" (result) \ + : "r" (I) \ + : "memory"); \ + \ + return result c_op I; \ +} + +#define ATOMIC_FETCH_OP(op, I, asm_op, mb, suffix) \ +static inline int arch_atomic_fetch_##op##suffix(int i, atomic_t *v) \ +{ \ + int result; \ + \ + __asm__ __volatile__( \ + "am"#asm_op#mb".w" " %1, %2, %0 \n" \ + : "+ZB" (v->counter), "=&r" (result) \ + : "r" (I) \ + : "memory"); \ + \ + return result; \ +} + +#define ATOMIC_OPS(op, I, asm_op, c_op) \ + ATOMIC_OP(op, I, asm_op) \ + ATOMIC_OP_RETURN(op, I, asm_op, c_op, _db, ) \ + ATOMIC_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \ + ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \ + ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed) + +ATOMIC_OPS(add, i, add, +) +ATOMIC_OPS(sub, -i, add, +) + +#define arch_atomic_add_return arch_atomic_add_return +#define arch_atomic_add_return_acquire arch_atomic_add_return +#define arch_atomic_add_return_release arch_atomic_add_return +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_sub_return arch_atomic_sub_return +#define arch_atomic_sub_return_acquire arch_atomic_sub_return +#define arch_atomic_sub_return_release arch_atomic_sub_return +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed +#define arch_atomic_fetch_add arch_atomic_fetch_add +#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add +#define arch_atomic_fetch_add_release arch_atomic_fetch_add +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_sub arch_atomic_fetch_sub +#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub +#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed + +#undef ATOMIC_OPS + +#define ATOMIC_OPS(op, I, asm_op) \ + ATOMIC_OP(op, I, asm_op) \ + ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \ + ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed) + +ATOMIC_OPS(and, i, and) +ATOMIC_OPS(or, i, or) +ATOMIC_OPS(xor, i, xor) + +#define arch_atomic_fetch_and arch_atomic_fetch_and +#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and +#define arch_atomic_fetch_and_release arch_atomic_fetch_and +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_or arch_atomic_fetch_or +#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or +#define arch_atomic_fetch_or_release arch_atomic_fetch_or +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_xor arch_atomic_fetch_xor +#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor +#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed + +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP +#undef ATOMIC_OP_RETURN +#undef ATOMIC_OP + +#ifdef CONFIG_64BIT + +#define ATOMIC64_OP(op, I, asm_op) \ +static inline void arch_atomic64_##op(long i, atomic64_t *v) \ +{ \ + __asm__ __volatile__( \ + "am"#asm_op".d " " $zero, %1, %0 \n" \ + : "+ZB" (v->counter) \ + : "r" (I) \ + : "memory"); \ +} + +#define ATOMIC64_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \ +static inline long arch_atomic64_##op##_return##suffix(long i, atomic64_t *v) \ +{ \ + long result; \ + __asm__ __volatile__( \ + "am"#asm_op#mb".d " " %1, %2, %0 \n" \ + : "+ZB" (v->counter), "=&r" (result) \ + : "r" (I) \ + : "memory"); \ + \ + return result c_op I; \ +} + +#define ATOMIC64_FETCH_OP(op, I, asm_op, mb, suffix) \ +static inline long arch_atomic64_fetch_##op##suffix(long i, atomic64_t *v) \ +{ \ + long result; \ + \ + __asm__ __volatile__( \ + "am"#asm_op#mb".d " " %1, %2, %0 \n" \ + : "+ZB" (v->counter), "=&r" (result) \ + : "r" (I) \ + : "memory"); \ + \ + return result; \ +} + +#define ATOMIC64_OPS(op, I, asm_op, c_op) \ + ATOMIC64_OP(op, I, asm_op) \ + ATOMIC64_OP_RETURN(op, I, asm_op, c_op, _db, ) \ + ATOMIC64_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \ + ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \ + ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed) + +ATOMIC64_OPS(add, i, add, +) +ATOMIC64_OPS(sub, -i, add, +) + +#define arch_atomic64_add_return arch_atomic64_add_return +#define arch_atomic64_add_return_acquire arch_atomic64_add_return +#define arch_atomic64_add_return_release arch_atomic64_add_return +#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed +#define arch_atomic64_sub_return arch_atomic64_sub_return +#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return +#define arch_atomic64_sub_return_release arch_atomic64_sub_return +#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed +#define arch_atomic64_fetch_add arch_atomic64_fetch_add +#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add +#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add +#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed +#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub +#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub +#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub +#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed + +#undef ATOMIC64_OPS + +#define ATOMIC64_OPS(op, I, asm_op) \ + ATOMIC64_OP(op, I, asm_op) \ + ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \ + ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed) + +ATOMIC64_OPS(and, i, and) +ATOMIC64_OPS(or, i, or) +ATOMIC64_OPS(xor, i, xor) + +#define arch_atomic64_fetch_and arch_atomic64_fetch_and +#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and +#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and +#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed +#define arch_atomic64_fetch_or arch_atomic64_fetch_or +#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or +#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or +#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed +#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor +#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor +#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor +#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed + +#undef ATOMIC64_OPS +#undef ATOMIC64_FETCH_OP +#undef ATOMIC64_OP_RETURN +#undef ATOMIC64_OP + +#endif + +#endif /* _ASM_ATOMIC_AMO_H */ diff --git a/arch/loongarch/include/asm/atomic-llsc.h b/arch/loongarch/include/asm/atomic-llsc.h new file mode 100644 index 000000000000..3ce500c88272 --- /dev/null +++ b/arch/loongarch/include/asm/atomic-llsc.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Atomic operations (LLSC). + * + * Copyright (C) 2024-2025 Loongson Technology Corporation Limited + */ + +#ifndef _ASM_ATOMIC_LLSC_H +#define _ASM_ATOMIC_LLSC_H + +#include <linux/types.h> +#include <asm/barrier.h> +#include <asm/cmpxchg.h> + +#define ATOMIC_OP(op, I, asm_op) \ +static inline void arch_atomic_##op(int i, atomic_t *v) \ +{ \ + int temp; \ + \ + __asm__ __volatile__( \ + "1: ll.w %0, %1 #atomic_" #op " \n" \ + " " #asm_op " %0, %0, %2 \n" \ + " sc.w %0, %1 \n" \ + " beq %0, $r0, 1b \n" \ + :"=&r" (temp) , "+ZC"(v->counter) \ + :"r" (I) \ + ); \ +} + +#define ATOMIC_OP_RETURN(op, I, asm_op) \ +static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \ +{ \ + int result, temp; \ + \ + __asm__ __volatile__( \ + "1: ll.w %1, %2 # atomic_" #op "_return \n" \ + " " #asm_op " %0, %1, %3 \n" \ + " sc.w %0, %2 \n" \ + " beq %0, $r0 ,1b \n" \ + " " #asm_op " %0, %1, %3 \n" \ + : "=&r" (result), "=&r" (temp), "+ZC"(v->counter) \ + : "r" (I)); \ + \ + return result; \ +} + +#define ATOMIC_FETCH_OP(op, I, asm_op) \ +static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ +{ \ + int result, temp; \ + \ + __asm__ __volatile__( \ + "1: ll.w %1, %2 # atomic_fetch_" #op " \n" \ + " " #asm_op " %0, %1, %3 \n" \ + " sc.w %0, %2 \n" \ + " beq %0, $r0 ,1b \n" \ + " add.w %0, %1 ,$r0 \n" \ + : "=&r" (result), "=&r" (temp), "+ZC" (v->counter) \ + : "r" (I)); \ + \ + return result; \ +} + +#define ATOMIC_OPS(op,I ,asm_op, c_op) \ + ATOMIC_OP(op, I, asm_op) \ + ATOMIC_OP_RETURN(op, I , asm_op) \ + ATOMIC_FETCH_OP(op, I, asm_op) + +ATOMIC_OPS(add, i , add.w ,+=) +ATOMIC_OPS(sub, -i , add.w ,+=) + +#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed + +#undef ATOMIC_OPS + +#define ATOMIC_OPS(op, I, asm_op) \ + ATOMIC_OP(op, I, asm_op) \ + ATOMIC_FETCH_OP(op, I, asm_op) + +ATOMIC_OPS(and, i, and) +ATOMIC_OPS(or, i, or) +ATOMIC_OPS(xor, i, xor) + +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed + +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP +#undef ATOMIC_OP_RETURN +#undef ATOMIC_OP + +#ifdef CONFIG_64BIT +#error "64-bit LLSC atomic operations are not supported" +#endif + +#endif /* _ASM_ATOMIC_LLSC_H */ diff --git a/arch/loongarch/include/asm/atomic.h b/arch/loongarch/include/asm/atomic.h index c86f0ab922ec..444b9ddcd004 100644 --- a/arch/loongarch/include/asm/atomic.h +++ b/arch/loongarch/include/asm/atomic.h @@ -11,6 +11,16 @@ #include <asm/barrier.h> #include <asm/cmpxchg.h> +#ifdef CONFIG_CPU_HAS_AMO +#include <asm/atomic-amo.h> +#else +#include <asm/atomic-llsc.h> +#endif + +#ifdef CONFIG_GENERIC_ATOMIC64 +#include <asm-generic/atomic64.h> +#endif + #if __SIZEOF_LONG__ == 4 #define __LL "ll.w " #define __SC "sc.w " @@ -34,100 +44,6 @@ #define arch_atomic_read(v) READ_ONCE((v)->counter) #define arch_atomic_set(v, i) WRITE_ONCE((v)->counter, (i)) -#define ATOMIC_OP(op, I, asm_op) \ -static inline void arch_atomic_##op(int i, atomic_t *v) \ -{ \ - __asm__ __volatile__( \ - "am"#asm_op".w" " $zero, %1, %0 \n" \ - : "+ZB" (v->counter) \ - : "r" (I) \ - : "memory"); \ -} - -#define ATOMIC_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \ -static inline int arch_atomic_##op##_return##suffix(int i, atomic_t *v) \ -{ \ - int result; \ - \ - __asm__ __volatile__( \ - "am"#asm_op#mb".w" " %1, %2, %0 \n" \ - : "+ZB" (v->counter), "=&r" (result) \ - : "r" (I) \ - : "memory"); \ - \ - return result c_op I; \ -} - -#define ATOMIC_FETCH_OP(op, I, asm_op, mb, suffix) \ -static inline int arch_atomic_fetch_##op##suffix(int i, atomic_t *v) \ -{ \ - int result; \ - \ - __asm__ __volatile__( \ - "am"#asm_op#mb".w" " %1, %2, %0 \n" \ - : "+ZB" (v->counter), "=&r" (result) \ - : "r" (I) \ - : "memory"); \ - \ - return result; \ -} - -#define ATOMIC_OPS(op, I, asm_op, c_op) \ - ATOMIC_OP(op, I, asm_op) \ - ATOMIC_OP_RETURN(op, I, asm_op, c_op, _db, ) \ - ATOMIC_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \ - ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \ - ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed) - -ATOMIC_OPS(add, i, add, +) -ATOMIC_OPS(sub, -i, add, +) - -#define arch_atomic_add_return arch_atomic_add_return -#define arch_atomic_add_return_acquire arch_atomic_add_return -#define arch_atomic_add_return_release arch_atomic_add_return -#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed -#define arch_atomic_sub_return arch_atomic_sub_return -#define arch_atomic_sub_return_acquire arch_atomic_sub_return -#define arch_atomic_sub_return_release arch_atomic_sub_return -#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed -#define arch_atomic_fetch_add arch_atomic_fetch_add -#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add -#define arch_atomic_fetch_add_release arch_atomic_fetch_add -#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed -#define arch_atomic_fetch_sub arch_atomic_fetch_sub -#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub -#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub -#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed - -#undef ATOMIC_OPS - -#define ATOMIC_OPS(op, I, asm_op) \ - ATOMIC_OP(op, I, asm_op) \ - ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \ - ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed) - -ATOMIC_OPS(and, i, and) -ATOMIC_OPS(or, i, or) -ATOMIC_OPS(xor, i, xor) - -#define arch_atomic_fetch_and arch_atomic_fetch_and -#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and -#define arch_atomic_fetch_and_release arch_atomic_fetch_and -#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed -#define arch_atomic_fetch_or arch_atomic_fetch_or -#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or -#define arch_atomic_fetch_or_release arch_atomic_fetch_or -#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed -#define arch_atomic_fetch_xor arch_atomic_fetch_xor -#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor -#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor -#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed - -#undef ATOMIC_OPS -#undef ATOMIC_FETCH_OP -#undef ATOMIC_OP_RETURN -#undef ATOMIC_OP - static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) { int prev, rc; @@ -194,99 +110,6 @@ static inline int arch_atomic_sub_if_positive(int i, atomic_t *v) #define arch_atomic64_read(v) READ_ONCE((v)->counter) #define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i)) -#define ATOMIC64_OP(op, I, asm_op) \ -static inline void arch_atomic64_##op(long i, atomic64_t *v) \ -{ \ - __asm__ __volatile__( \ - "am"#asm_op".d " " $zero, %1, %0 \n" \ - : "+ZB" (v->counter) \ - : "r" (I) \ - : "memory"); \ -} - -#define ATOMIC64_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \ -static inline long arch_atomic64_##op##_return##suffix(long i, atomic64_t *v) \ -{ \ - long result; \ - __asm__ __volatile__( \ - "am"#asm_op#mb".d " " %1, %2, %0 \n" \ - : "+ZB" (v->counter), "=&r" (result) \ - : "r" (I) \ - : "memory"); \ - \ - return result c_op I; \ -} - -#define ATOMIC64_FETCH_OP(op, I, asm_op, mb, suffix) \ -static inline long arch_atomic64_fetch_##op##suffix(long i, atomic64_t *v) \ -{ \ - long result; \ - \ - __asm__ __volatile__( \ - "am"#asm_op#mb".d " " %1, %2, %0 \n" \ - : "+ZB" (v->counter), "=&r" (result) \ - : "r" (I) \ - : "memory"); \ - \ - return result; \ -} - -#define ATOMIC64_OPS(op, I, asm_op, c_op) \ - ATOMIC64_OP(op, I, asm_op) \ - ATOMIC64_OP_RETURN(op, I, asm_op, c_op, _db, ) \ - ATOMIC64_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \ - ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \ - ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed) - -ATOMIC64_OPS(add, i, add, +) -ATOMIC64_OPS(sub, -i, add, +) - -#define arch_atomic64_add_return arch_atomic64_add_return -#define arch_atomic64_add_return_acquire arch_atomic64_add_return -#define arch_atomic64_add_return_release arch_atomic64_add_return -#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed -#define arch_atomic64_sub_return arch_atomic64_sub_return -#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return -#define arch_atomic64_sub_return_release arch_atomic64_sub_return -#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed -#define arch_atomic64_fetch_add arch_atomic64_fetch_add -#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add -#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add -#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed -#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub -#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub -#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub -#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed - -#undef ATOMIC64_OPS - -#define ATOMIC64_OPS(op, I, asm_op) \ - ATOMIC64_OP(op, I, asm_op) \ - ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \ - ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed) - -ATOMIC64_OPS(and, i, and) -ATOMIC64_OPS(or, i, or) -ATOMIC64_OPS(xor, i, xor) - -#define arch_atomic64_fetch_and arch_atomic64_fetch_and -#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and -#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and -#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed -#define arch_atomic64_fetch_or arch_atomic64_fetch_or -#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or -#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or -#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed -#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor -#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor -#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor -#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed - -#undef ATOMIC64_OPS -#undef ATOMIC64_FETCH_OP -#undef ATOMIC64_OP_RETURN -#undef ATOMIC64_OP - static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u) { long prev, rc; diff --git a/arch/loongarch/include/asm/bitops.h b/arch/loongarch/include/asm/bitops.h index 69e00f8d8034..411106bf9902 100644 --- a/arch/loongarch/include/asm/bitops.h +++ b/arch/loongarch/include/asm/bitops.h @@ -13,11 +13,22 @@ #include <asm/barrier.h> +#ifdef CONFIG_32BIT_REDUCED + +#include <asm-generic/bitops/ffs.h> +#include <asm-generic/bitops/fls.h> +#include <asm-generic/bitops/__ffs.h> +#include <asm-generic/bitops/__fls.h> + +#else /* CONFIG_32BIT_STANDARD || CONFIG_64BIT */ + #include <asm-generic/bitops/builtin-ffs.h> #include <asm-generic/bitops/builtin-fls.h> #include <asm-generic/bitops/builtin-__ffs.h> #include <asm-generic/bitops/builtin-__fls.h> +#endif + #include <asm-generic/bitops/ffz.h> #include <asm-generic/bitops/fls64.h> diff --git a/arch/loongarch/include/asm/bitrev.h b/arch/loongarch/include/asm/bitrev.h index 46f275b9cdf7..757738ea38d7 100644 --- a/arch/loongarch/include/asm/bitrev.h +++ b/arch/loongarch/include/asm/bitrev.h @@ -11,7 +11,7 @@ static __always_inline __attribute_const__ u32 __arch_bitrev32(u32 x) { u32 ret; - asm("bitrev.4b %0, %1" : "=r"(ret) : "r"(__swab32(x))); + asm("bitrev.w %0, %1" : "=r"(ret) : "r"(x)); return ret; } diff --git a/arch/loongarch/include/asm/checksum.h b/arch/loongarch/include/asm/checksum.h index cabbf6af44c4..cc2754e0aa25 100644 --- a/arch/loongarch/include/asm/checksum.h +++ b/arch/loongarch/include/asm/checksum.h @@ -9,6 +9,8 @@ #include <linux/bitops.h> #include <linux/in6.h> +#ifdef CONFIG_64BIT + #define _HAVE_ARCH_IPV6_CSUM __sum16 csum_ipv6_magic(const struct in6_addr *saddr, const struct in6_addr *daddr, @@ -61,6 +63,8 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) extern unsigned int do_csum(const unsigned char *buff, int len); #define do_csum do_csum +#endif + #include <asm-generic/checksum.h> #endif /* __ASM_CHECKSUM_H */ diff --git a/arch/loongarch/include/asm/cmpxchg.h b/arch/loongarch/include/asm/cmpxchg.h index 979fde61bba8..0494c2ab553e 100644 --- a/arch/loongarch/include/asm/cmpxchg.h +++ b/arch/loongarch/include/asm/cmpxchg.h @@ -9,17 +9,33 @@ #include <linux/build_bug.h> #include <asm/barrier.h> -#define __xchg_asm(amswap_db, m, val) \ +#define __xchg_amo_asm(amswap_db, m, val) \ ({ \ - __typeof(val) __ret; \ + __typeof(val) __ret; \ \ - __asm__ __volatile__ ( \ - " "amswap_db" %1, %z2, %0 \n" \ - : "+ZB" (*m), "=&r" (__ret) \ - : "Jr" (val) \ - : "memory"); \ + __asm__ __volatile__ ( \ + " "amswap_db" %1, %z2, %0 \n" \ + : "+ZB" (*m), "=&r" (__ret) \ + : "Jr" (val) \ + : "memory"); \ \ - __ret; \ + __ret; \ +}) + +#define __xchg_llsc_asm(ld, st, m, val) \ +({ \ + __typeof(val) __ret, __tmp; \ + \ + asm volatile ( \ + "1: ll.w %0, %3 \n" \ + " move %1, %z4 \n" \ + " sc.w %1, %2 \n" \ + " beqz %1, 1b \n" \ + : "=&r" (__ret), "=&r" (__tmp), "=ZC" (*m) \ + : "ZC" (*m), "Jr" (val) \ + : "memory"); \ + \ + __ret; \ }) static inline unsigned int __xchg_small(volatile void *ptr, unsigned int val, @@ -67,13 +83,23 @@ __arch_xchg(volatile void *ptr, unsigned long x, int size) switch (size) { case 1: case 2: - return __xchg_small(ptr, x, size); + return __xchg_small((volatile void *)ptr, x, size); case 4: - return __xchg_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x); +#ifdef CONFIG_CPU_HAS_AMO + return __xchg_amo_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x); +#else + return __xchg_llsc_asm("ll.w", "sc.w", (volatile u32 *)ptr, (u32)x); +#endif /* CONFIG_CPU_HAS_AMO */ +#ifdef CONFIG_64BIT case 8: - return __xchg_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x); +#ifdef CONFIG_CPU_HAS_AMO + return __xchg_amo_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x); +#else + return __xchg_llsc_asm("ll.d", "sc.d", (volatile u64 *)ptr, (u64)x); +#endif /* CONFIG_CPU_HAS_AMO */ +#endif /* CONFIG_64BIT */ default: BUILD_BUG(); diff --git a/arch/loongarch/include/asm/cpu-features.h b/arch/loongarch/include/asm/cpu-features.h index bd5f0457ad21..3745d991a99a 100644 --- a/arch/loongarch/include/asm/cpu-features.h +++ b/arch/loongarch/include/asm/cpu-features.h @@ -20,16 +20,13 @@ #define cpu_has_loongarch64 (cpu_data[0].isa_level & LOONGARCH_CPU_ISA_64BIT) #ifdef CONFIG_32BIT -# define cpu_has_64bits (cpu_data[0].isa_level & LOONGARCH_CPU_ISA_64BIT) # define cpu_vabits 31 # define cpu_pabits 31 #endif #ifdef CONFIG_64BIT -# define cpu_has_64bits 1 # define cpu_vabits cpu_data[0].vabits # define cpu_pabits cpu_data[0].pabits -# define __NEED_ADDRBITS_PROBE #endif /* diff --git a/arch/loongarch/include/asm/dmi.h b/arch/loongarch/include/asm/dmi.h index 605493417753..11bb3c8a7179 100644 --- a/arch/loongarch/include/asm/dmi.h +++ b/arch/loongarch/include/asm/dmi.h @@ -12,7 +12,7 @@ #define dmi_early_unmap(x, l) dmi_unmap(x) #define dmi_alloc(l) memblock_alloc(l, PAGE_SIZE) -static inline void *dmi_remap(u64 phys_addr, unsigned long size) +static inline void *dmi_remap(phys_addr_t phys_addr, unsigned long size) { return ((void *)TO_CACHE(phys_addr)); } diff --git a/arch/loongarch/include/asm/elf.h b/arch/loongarch/include/asm/elf.h index f16bd42456e4..912c50cdd6b7 100644 --- a/arch/loongarch/include/asm/elf.h +++ b/arch/loongarch/include/asm/elf.h @@ -120,6 +120,36 @@ #define R_LARCH_ADD_ULEB128 107 #define R_LARCH_SUB_ULEB128 108 #define R_LARCH_64_PCREL 109 +#define R_LARCH_CALL36 110 +#define R_LARCH_TLS_DESC_PC_HI20 111 +#define R_LARCH_TLS_DESC_PC_LO12 112 +#define R_LARCH_TLS_DESC64_PC_LO20 113 +#define R_LARCH_TLS_DESC64_PC_HI12 114 +#define R_LARCH_TLS_DESC_HI20 115 +#define R_LARCH_TLS_DESC_LO12 116 +#define R_LARCH_TLS_DESC64_LO20 117 +#define R_LARCH_TLS_DESC64_HI12 118 +#define R_LARCH_TLS_DESC_LD 119 +#define R_LARCH_TLS_DESC_CALL 120 +#define R_LARCH_TLS_LE_HI20_R 121 +#define R_LARCH_TLS_LE_ADD_R 122 +#define R_LARCH_TLS_LE_LO12_R 123 +#define R_LARCH_TLS_LD_PCREL20_S2 124 +#define R_LARCH_TLS_GD_PCREL20_S2 125 +#define R_LARCH_TLS_DESC_PCREL20_S2 126 +#define R_LARCH_CALL30 127 +#define R_LARCH_PCADD_HI20 128 +#define R_LARCH_PCADD_LO12 129 +#define R_LARCH_GOT_PCADD_HI20 130 +#define R_LARCH_GOT_PCADD_LO12 131 +#define R_LARCH_TLS_IE_PCADD_HI20 132 +#define R_LARCH_TLS_IE_PCADD_LO12 133 +#define R_LARCH_TLS_LD_PCADD_HI20 134 +#define R_LARCH_TLS_LD_PCADD_LO12 135 +#define R_LARCH_TLS_GD_PCADD_HI20 136 +#define R_LARCH_TLS_GD_PCADD_LO12 137 +#define R_LARCH_TLS_DESC_PCADD_HI20 138 +#define R_LARCH_TLS_DESC_PCADD_LO12 139 #ifndef ELF_ARCH @@ -156,6 +186,7 @@ typedef elf_greg_t elf_gregset_t[ELF_NGREG]; typedef double elf_fpreg_t; typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; +void loongarch_dump_regs32(u32 *uregs, const struct pt_regs *regs); void loongarch_dump_regs64(u64 *uregs, const struct pt_regs *regs); #ifdef CONFIG_32BIT diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 55e64a12a124..f9f207082d0e 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -438,8 +438,10 @@ static inline bool is_branch_ins(union loongarch_instruction *ip) static inline bool is_ra_save_ins(union loongarch_instruction *ip) { - /* st.d $ra, $sp, offset */ - return ip->reg2i12_format.opcode == std_op && + const u32 opcode = IS_ENABLED(CONFIG_32BIT) ? stw_op : std_op; + + /* st.w / st.d $ra, $sp, offset */ + return ip->reg2i12_format.opcode == opcode && ip->reg2i12_format.rj == LOONGARCH_GPR_SP && ip->reg2i12_format.rd == LOONGARCH_GPR_RA && !is_imm12_negative(ip->reg2i12_format.immediate); @@ -447,8 +449,10 @@ static inline bool is_ra_save_ins(union loongarch_instruction *ip) static inline bool is_stack_alloc_ins(union loongarch_instruction *ip) { - /* addi.d $sp, $sp, -imm */ - return ip->reg2i12_format.opcode == addid_op && + const u32 opcode = IS_ENABLED(CONFIG_32BIT) ? addiw_op : addid_op; + + /* addi.w / addi.d $sp, $sp, -imm */ + return ip->reg2i12_format.opcode == opcode && ip->reg2i12_format.rj == LOONGARCH_GPR_SP && ip->reg2i12_format.rd == LOONGARCH_GPR_SP && is_imm12_negative(ip->reg2i12_format.immediate); diff --git a/arch/loongarch/include/asm/irq.h b/arch/loongarch/include/asm/irq.h index 12bd15578c33..3943647503a9 100644 --- a/arch/loongarch/include/asm/irq.h +++ b/arch/loongarch/include/asm/irq.h @@ -50,10 +50,22 @@ void spurious_interrupt(void); #define NR_LEGACY_VECTORS 16 #define IRQ_MATRIX_BITS NR_VECTORS +#define AVEC_IRQ_SHIFT 4 +#define AVEC_IRQ_BIT 8 +#define AVEC_IRQ_MASK GENMASK(AVEC_IRQ_BIT - 1, 0) +#define AVEC_CPU_SHIFT 12 +#define AVEC_CPU_BIT 16 +#define AVEC_CPU_MASK GENMASK(AVEC_CPU_BIT - 1, 0) + #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace void arch_trigger_cpumask_backtrace(const struct cpumask *mask, int exclude_cpu); +#ifdef CONFIG_32BIT +#define MAX_IO_PICS 1 +#else #define MAX_IO_PICS 8 +#endif + #define NR_IRQS (64 + NR_VECTORS * (NR_CPUS + MAX_IO_PICS)) struct acpi_vector_group { diff --git a/arch/loongarch/include/asm/jump_label.h b/arch/loongarch/include/asm/jump_label.h index 4000c7603d8e..dcaecf69ea5a 100644 --- a/arch/loongarch/include/asm/jump_label.h +++ b/arch/loongarch/include/asm/jump_label.h @@ -10,15 +10,23 @@ #ifndef __ASSEMBLER__ #include <linux/types.h> +#include <linux/stringify.h> +#include <asm/asm.h> #define JUMP_LABEL_NOP_SIZE 4 +#ifdef CONFIG_32BIT +#define JUMP_LABEL_TYPE ".long " +#else +#define JUMP_LABEL_TYPE ".quad " +#endif + /* This macro is also expanded on the Rust side. */ #define JUMP_TABLE_ENTRY(key, label) \ ".pushsection __jump_table, \"aw\" \n\t" \ - ".align 3 \n\t" \ + ".align " __stringify(PTRLOG) " \n\t" \ ".long 1b - ., " label " - . \n\t" \ - ".quad " key " - . \n\t" \ + JUMP_LABEL_TYPE key " - . \n\t" \ ".popsection \n\t" #define ARCH_STATIC_BRANCH_ASM(key, label) \ diff --git a/arch/loongarch/include/asm/local.h b/arch/loongarch/include/asm/local.h index f53ea653af76..2d118b1a060e 100644 --- a/arch/loongarch/include/asm/local.h +++ b/arch/loongarch/include/asm/local.h @@ -8,6 +8,7 @@ #include <linux/percpu.h> #include <linux/bitops.h> #include <linux/atomic.h> +#include <asm/asm.h> #include <asm/cmpxchg.h> typedef struct { @@ -27,6 +28,7 @@ typedef struct { /* * Same as above, but return the result value */ +#ifdef CONFIG_CPU_HAS_AMO static inline long local_add_return(long i, local_t *l) { unsigned long result; @@ -55,6 +57,41 @@ static inline long local_sub_return(long i, local_t *l) return result; } +#else +static inline long local_add_return(long i, local_t *l) +{ + unsigned long result, temp; + + __asm__ __volatile__( + "1:" __LL "%1, %2 # local_add_return \n" + __stringify(LONG_ADD) " %0, %1, %3 \n" + __SC "%0, %2 \n" + " beq %0, $r0, 1b \n" + __stringify(LONG_ADD) " %0, %1, %3 \n" + : "=&r" (result), "=&r" (temp), "=ZC" (l->a.counter) + : "r" (i), "ZC" (l->a.counter) + : "memory"); + + return result; +} + +static inline long local_sub_return(long i, local_t *l) +{ + unsigned long result, temp; + + __asm__ __volatile__( + "1:" __LL "%1, %2 # local_sub_return \n" + __stringify(LONG_SUB) " %0, %1, %3 \n" + __SC "%0, %2 \n" + " beq %0, $r0, 1b \n" + __stringify(LONG_SUB) " %0, %1, %3 \n" + : "=&r" (result), "=&r" (temp), "=ZC" (l->a.counter) + : "r" (i), "ZC" (l->a.counter) + : "memory"); + + return result; +} +#endif static inline long local_cmpxchg(local_t *l, long old, long new) { diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index 58a4a3b6b035..e6b8ff61c8cc 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -182,6 +182,16 @@ #define csr_xchg32(val, mask, reg) __csrxchg_w(val, mask, reg) #define csr_xchg64(val, mask, reg) __csrxchg_d(val, mask, reg) +#ifdef CONFIG_32BIT +#define csr_read(reg) csr_read32(reg) +#define csr_write(val, reg) csr_write32(val, reg) +#define csr_xchg(val, mask, reg) csr_xchg32(val, mask, reg) +#else +#define csr_read(reg) csr_read64(reg) +#define csr_write(val, reg) csr_write64(val, reg) +#define csr_xchg(val, mask, reg) csr_xchg64(val, mask, reg) +#endif + /* IOCSR */ #define iocsr_read32(reg) __iocsrrd_w(reg) #define iocsr_read64(reg) __iocsrrd_d(reg) @@ -904,6 +914,26 @@ #define LOONGARCH_CSR_DMWIN3 0x183 /* 64 direct map win3: MEM */ /* Direct Map window 0/1/2/3 */ + +#ifdef CONFIG_32BIT + +#define CSR_DMW0_PLV0 (1 << 0) +#define CSR_DMW0_VSEG (0x4) +#define CSR_DMW0_BASE (CSR_DMW0_VSEG << DMW_PABITS) +#define CSR_DMW0_INIT (CSR_DMW0_BASE | CSR_DMW0_PLV0) + +#define CSR_DMW1_PLV0 (1 << 0) +#define CSR_DMW1_MAT (1 << 4) +#define CSR_DMW1_VSEG (0x5) +#define CSR_DMW1_BASE (CSR_DMW1_VSEG << DMW_PABITS) +#define CSR_DMW1_INIT (CSR_DMW1_BASE | CSR_DMW1_MAT | CSR_DMW1_PLV0) + +#define CSR_DMW2_INIT 0x0 + +#define CSR_DMW3_INIT 0x0 + +#else + #define CSR_DMW0_PLV0 _CONST64_(1 << 0) #define CSR_DMW0_VSEG _CONST64_(0x8000) #define CSR_DMW0_BASE (CSR_DMW0_VSEG << DMW_PABITS) @@ -923,6 +953,8 @@ #define CSR_DMW3_INIT 0x0 +#endif + /* Performance Counter registers */ #define LOONGARCH_CSR_PERFCTRL0 0x200 /* 32 perf event 0 config */ #define LOONGARCH_CSR_PERFCNTR0 0x201 /* 64 perf event 0 count value */ @@ -1208,7 +1240,35 @@ #ifndef __ASSEMBLER__ -static __always_inline u64 drdtime(void) +#ifdef CONFIG_32BIT + +static __always_inline u32 rdtime_h(void) +{ + u32 val = 0; + + __asm__ __volatile__( + "rdtimeh.w %0, $zero\n\t" + : "=r"(val) + : + ); + return val; +} + +static __always_inline u32 rdtime_l(void) +{ + u32 val = 0; + + __asm__ __volatile__( + "rdtimel.w %0, $zero\n\t" + : "=r"(val) + : + ); + return val; +} + +#else + +static __always_inline u64 rdtime_d(void) { u64 val = 0; @@ -1220,11 +1280,14 @@ static __always_inline u64 drdtime(void) return val; } +#endif + static inline unsigned int get_csr_cpuid(void) { return csr_read32(LOONGARCH_CSR_CPUID); } +#ifdef CONFIG_64BIT static inline void csr_any_send(unsigned int addr, unsigned int data, unsigned int data_mask, unsigned int cpu) { @@ -1236,6 +1299,7 @@ static inline void csr_any_send(unsigned int addr, unsigned int data, val |= ((uint64_t)data << IOCSR_ANY_SEND_BUF_SHIFT); iocsr_write64(val, LOONGARCH_IOCSR_ANY_SEND); } +#endif static inline unsigned int read_csr_excode(void) { @@ -1259,22 +1323,22 @@ static inline void write_csr_pagesize(unsigned int size) static inline unsigned int read_csr_tlbrefill_pagesize(void) { - return (csr_read64(LOONGARCH_CSR_TLBREHI) & CSR_TLBREHI_PS) >> CSR_TLBREHI_PS_SHIFT; + return (csr_read(LOONGARCH_CSR_TLBREHI) & CSR_TLBREHI_PS) >> CSR_TLBREHI_PS_SHIFT; } static inline void write_csr_tlbrefill_pagesize(unsigned int size) { - csr_xchg64(size << CSR_TLBREHI_PS_SHIFT, CSR_TLBREHI_PS, LOONGARCH_CSR_TLBREHI); + csr_xchg(size << CSR_TLBREHI_PS_SHIFT, CSR_TLBREHI_PS, LOONGARCH_CSR_TLBREHI); } #define read_csr_asid() csr_read32(LOONGARCH_CSR_ASID) #define write_csr_asid(val) csr_write32(val, LOONGARCH_CSR_ASID) -#define read_csr_entryhi() csr_read64(LOONGARCH_CSR_TLBEHI) -#define write_csr_entryhi(val) csr_write64(val, LOONGARCH_CSR_TLBEHI) -#define read_csr_entrylo0() csr_read64(LOONGARCH_CSR_TLBELO0) -#define write_csr_entrylo0(val) csr_write64(val, LOONGARCH_CSR_TLBELO0) -#define read_csr_entrylo1() csr_read64(LOONGARCH_CSR_TLBELO1) -#define write_csr_entrylo1(val) csr_write64(val, LOONGARCH_CSR_TLBELO1) +#define read_csr_entryhi() csr_read(LOONGARCH_CSR_TLBEHI) +#define write_csr_entryhi(val) csr_write(val, LOONGARCH_CSR_TLBEHI) +#define read_csr_entrylo0() csr_read(LOONGARCH_CSR_TLBELO0) +#define write_csr_entrylo0(val) csr_write(val, LOONGARCH_CSR_TLBELO0) +#define read_csr_entrylo1() csr_read(LOONGARCH_CSR_TLBELO1) +#define write_csr_entrylo1(val) csr_write(val, LOONGARCH_CSR_TLBELO1) #define read_csr_ecfg() csr_read32(LOONGARCH_CSR_ECFG) #define write_csr_ecfg(val) csr_write32(val, LOONGARCH_CSR_ECFG) #define read_csr_estat() csr_read32(LOONGARCH_CSR_ESTAT) @@ -1284,20 +1348,20 @@ static inline void write_csr_tlbrefill_pagesize(unsigned int size) #define read_csr_euen() csr_read32(LOONGARCH_CSR_EUEN) #define write_csr_euen(val) csr_write32(val, LOONGARCH_CSR_EUEN) #define read_csr_cpuid() csr_read32(LOONGARCH_CSR_CPUID) -#define read_csr_prcfg1() csr_read64(LOONGARCH_CSR_PRCFG1) -#define write_csr_prcfg1(val) csr_write64(val, LOONGARCH_CSR_PRCFG1) -#define read_csr_prcfg2() csr_read64(LOONGARCH_CSR_PRCFG2) -#define write_csr_prcfg2(val) csr_write64(val, LOONGARCH_CSR_PRCFG2) -#define read_csr_prcfg3() csr_read64(LOONGARCH_CSR_PRCFG3) -#define write_csr_prcfg3(val) csr_write64(val, LOONGARCH_CSR_PRCFG3) +#define read_csr_prcfg1() csr_read(LOONGARCH_CSR_PRCFG1) +#define write_csr_prcfg1(val) csr_write(val, LOONGARCH_CSR_PRCFG1) +#define read_csr_prcfg2() csr_read(LOONGARCH_CSR_PRCFG2) +#define write_csr_prcfg2(val) csr_write(val, LOONGARCH_CSR_PRCFG2) +#define read_csr_prcfg3() csr_read(LOONGARCH_CSR_PRCFG3) +#define write_csr_prcfg3(val) csr_write(val, LOONGARCH_CSR_PRCFG3) #define read_csr_stlbpgsize() csr_read32(LOONGARCH_CSR_STLBPGSIZE) #define write_csr_stlbpgsize(val) csr_write32(val, LOONGARCH_CSR_STLBPGSIZE) #define read_csr_rvacfg() csr_read32(LOONGARCH_CSR_RVACFG) #define write_csr_rvacfg(val) csr_write32(val, LOONGARCH_CSR_RVACFG) #define write_csr_tintclear(val) csr_write32(val, LOONGARCH_CSR_TINTCLR) -#define read_csr_impctl1() csr_read64(LOONGARCH_CSR_IMPCTL1) -#define write_csr_impctl1(val) csr_write64(val, LOONGARCH_CSR_IMPCTL1) -#define write_csr_impctl2(val) csr_write64(val, LOONGARCH_CSR_IMPCTL2) +#define read_csr_impctl1() csr_read(LOONGARCH_CSR_IMPCTL1) +#define write_csr_impctl1(val) csr_write(val, LOONGARCH_CSR_IMPCTL1) +#define write_csr_impctl2(val) csr_write(val, LOONGARCH_CSR_IMPCTL2) #define read_csr_perfctrl0() csr_read64(LOONGARCH_CSR_PERFCTRL0) #define read_csr_perfcntr0() csr_read64(LOONGARCH_CSR_PERFCNTR0) @@ -1378,8 +1442,10 @@ __BUILD_CSR_OP(tlbidx) #define ENTRYLO_C_SHIFT 4 #define ENTRYLO_C (_ULCAST_(3) << ENTRYLO_C_SHIFT) #define ENTRYLO_G (_ULCAST_(1) << 6) +#ifdef CONFIG_64BIT #define ENTRYLO_NR (_ULCAST_(1) << 61) #define ENTRYLO_NX (_ULCAST_(1) << 62) +#endif /* Values for PageSize register */ #define PS_4K 0x0000000c diff --git a/arch/loongarch/include/asm/module.h b/arch/loongarch/include/asm/module.h index f33f3fd32ecc..d56a968273de 100644 --- a/arch/loongarch/include/asm/module.h +++ b/arch/loongarch/include/asm/module.h @@ -38,8 +38,10 @@ struct got_entry { struct plt_entry { u32 inst_lu12iw; +#ifdef CONFIG_64BIT u32 inst_lu32id; u32 inst_lu52id; +#endif u32 inst_jirl; }; @@ -57,6 +59,14 @@ static inline struct got_entry emit_got_entry(Elf_Addr val) static inline struct plt_entry emit_plt_entry(unsigned long val) { +#ifdef CONFIG_32BIT + u32 lu12iw, jirl; + + lu12iw = larch_insn_gen_lu12iw(LOONGARCH_GPR_T1, ADDR_IMM(val, LU12IW)); + jirl = larch_insn_gen_jirl(0, LOONGARCH_GPR_T1, ADDR_IMM(val, ORI)); + + return (struct plt_entry) { lu12iw, jirl }; +#else u32 lu12iw, lu32id, lu52id, jirl; lu12iw = larch_insn_gen_lu12iw(LOONGARCH_GPR_T1, ADDR_IMM(val, LU12IW)); @@ -65,6 +75,7 @@ static inline struct plt_entry emit_plt_entry(unsigned long val) jirl = larch_insn_gen_jirl(0, LOONGARCH_GPR_T1, ADDR_IMM(val, ORI)); return (struct plt_entry) { lu12iw, lu32id, lu52id, jirl }; +#endif } static inline struct plt_idx_entry emit_plt_idx_entry(unsigned long val) diff --git a/arch/loongarch/include/asm/page.h b/arch/loongarch/include/asm/page.h index a3aaf34fba16..256d1ff7a1e3 100644 --- a/arch/loongarch/include/asm/page.h +++ b/arch/loongarch/include/asm/page.h @@ -10,7 +10,7 @@ #include <vdso/page.h> -#define HPAGE_SHIFT (PAGE_SHIFT + PAGE_SHIFT - 3) +#define HPAGE_SHIFT (PAGE_SHIFT + PAGE_SHIFT - PTRLOG) #define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) #define HPAGE_MASK (~(HPAGE_SIZE - 1)) #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) diff --git a/arch/loongarch/include/asm/percpu.h b/arch/loongarch/include/asm/percpu.h index 87be9b14e9da..583f2466262f 100644 --- a/arch/loongarch/include/asm/percpu.h +++ b/arch/loongarch/include/asm/percpu.h @@ -13,7 +13,7 @@ * the loading address of main kernel image, but far from where the modules are * loaded. Tell the compiler this fact when using explicit relocs. */ -#if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS) +#if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS) && defined(CONFIG_64BIT) # if __has_attribute(model) # define PER_CPU_ATTRIBUTES __attribute__((model("extreme"))) # else @@ -27,7 +27,7 @@ register unsigned long __my_cpu_offset __asm__("$r21"); static inline void set_my_cpu_offset(unsigned long off) { __my_cpu_offset = off; - csr_write64(off, PERCPU_BASE_KS); + csr_write(off, PERCPU_BASE_KS); } #define __my_cpu_offset \ @@ -36,6 +36,8 @@ static inline void set_my_cpu_offset(unsigned long off) __my_cpu_offset; \ }) +#ifdef CONFIG_CPU_HAS_AMO + #define PERCPU_OP(op, asm_op, c_op) \ static __always_inline unsigned long __percpu_##op(void *ptr, \ unsigned long val, int size) \ @@ -68,25 +70,9 @@ PERCPU_OP(and, and, &) PERCPU_OP(or, or, |) #undef PERCPU_OP -static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, int size) -{ - switch (size) { - case 1: - case 2: - return __xchg_small((volatile void *)ptr, val, size); - - case 4: - return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val); - - case 8: - return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val); +#endif - default: - BUILD_BUG(); - } - - return 0; -} +#ifdef CONFIG_64BIT #define __pcpu_op_1(op) op ".b " #define __pcpu_op_2(op) op ".h " @@ -115,6 +101,10 @@ do { \ : "memory"); \ } while (0) +#endif + +#define __percpu_xchg __arch_xchg + /* this_cpu_cmpxchg */ #define _protect_cmpxchg_local(pcp, o, n) \ ({ \ @@ -135,6 +125,8 @@ do { \ __retval; \ }) +#ifdef CONFIG_CPU_HAS_AMO + #define _percpu_add(pcp, val) \ _pcp_protect(__percpu_add, pcp, val) @@ -146,9 +138,6 @@ do { \ #define _percpu_or(pcp, val) \ _pcp_protect(__percpu_or, pcp, val) -#define _percpu_xchg(pcp, val) ((typeof(pcp)) \ - _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val))) - #define this_cpu_add_4(pcp, val) _percpu_add(pcp, val) #define this_cpu_add_8(pcp, val) _percpu_add(pcp, val) @@ -161,6 +150,10 @@ do { \ #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val) #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val) +#endif + +#ifdef CONFIG_64BIT + #define this_cpu_read_1(pcp) _percpu_read(1, pcp) #define this_cpu_read_2(pcp) _percpu_read(2, pcp) #define this_cpu_read_4(pcp) _percpu_read(4, pcp) @@ -171,6 +164,11 @@ do { \ #define this_cpu_write_4(pcp, val) _percpu_write(4, pcp, val) #define this_cpu_write_8(pcp, val) _percpu_write(8, pcp, val) +#endif + +#define _percpu_xchg(pcp, val) ((typeof(pcp)) \ + _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val))) + #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val) #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val) #define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val) diff --git a/arch/loongarch/include/asm/pgtable-bits.h b/arch/loongarch/include/asm/pgtable-bits.h index 2fc3789220ac..b565573cd82e 100644 --- a/arch/loongarch/include/asm/pgtable-bits.h +++ b/arch/loongarch/include/asm/pgtable-bits.h @@ -6,6 +6,26 @@ #define _ASM_PGTABLE_BITS_H /* Page table bits */ + +#ifdef CONFIG_32BIT +#define _PAGE_VALID_SHIFT 0 +#define _PAGE_ACCESSED_SHIFT 0 /* Reuse Valid for Accessed */ +#define _PAGE_DIRTY_SHIFT 1 +#define _PAGE_PLV_SHIFT 2 /* 2~3, two bits */ +#define _CACHE_SHIFT 4 /* 4~5, two bits */ +#define _PAGE_GLOBAL_SHIFT 6 +#define _PAGE_HUGE_SHIFT 6 /* HUGE is a PMD bit */ +#define _PAGE_PRESENT_SHIFT 7 +#define _PAGE_PFN_SHIFT 8 +#define _PAGE_HGLOBAL_SHIFT 12 /* HGlobal is a PMD bit */ +#define _PAGE_SWP_EXCLUSIVE_SHIFT 13 +#define _PAGE_PFN_END_SHIFT 28 +#define _PAGE_WRITE_SHIFT 29 +#define _PAGE_MODIFIED_SHIFT 30 +#define _PAGE_PRESENT_INVALID_SHIFT 31 +#endif + +#ifdef CONFIG_64BIT #define _PAGE_VALID_SHIFT 0 #define _PAGE_ACCESSED_SHIFT 0 /* Reuse Valid for Accessed */ #define _PAGE_DIRTY_SHIFT 1 @@ -18,14 +38,15 @@ #define _PAGE_MODIFIED_SHIFT 9 #define _PAGE_PROTNONE_SHIFT 10 #define _PAGE_SPECIAL_SHIFT 11 -#define _PAGE_HGLOBAL_SHIFT 12 /* HGlobal is a PMD bit */ #define _PAGE_PFN_SHIFT 12 +#define _PAGE_HGLOBAL_SHIFT 12 /* HGlobal is a PMD bit */ #define _PAGE_SWP_EXCLUSIVE_SHIFT 23 #define _PAGE_PFN_END_SHIFT 48 #define _PAGE_PRESENT_INVALID_SHIFT 60 #define _PAGE_NO_READ_SHIFT 61 #define _PAGE_NO_EXEC_SHIFT 62 #define _PAGE_RPLV_SHIFT 63 +#endif /* Used by software */ #define _PAGE_PRESENT (_ULCAST_(1) << _PAGE_PRESENT_SHIFT) @@ -33,10 +54,15 @@ #define _PAGE_WRITE (_ULCAST_(1) << _PAGE_WRITE_SHIFT) #define _PAGE_ACCESSED (_ULCAST_(1) << _PAGE_ACCESSED_SHIFT) #define _PAGE_MODIFIED (_ULCAST_(1) << _PAGE_MODIFIED_SHIFT) +#ifdef CONFIG_32BIT +#define _PAGE_PROTNONE 0 +#define _PAGE_SPECIAL 0 +#else #define _PAGE_PROTNONE (_ULCAST_(1) << _PAGE_PROTNONE_SHIFT) #define _PAGE_SPECIAL (_ULCAST_(1) << _PAGE_SPECIAL_SHIFT) +#endif -/* We borrow bit 23 to store the exclusive marker in swap PTEs. */ +/* We borrow bit 13/23 to store the exclusive marker in swap PTEs. */ #define _PAGE_SWP_EXCLUSIVE (_ULCAST_(1) << _PAGE_SWP_EXCLUSIVE_SHIFT) /* Used by TLB hardware (placed in EntryLo*) */ @@ -46,9 +72,15 @@ #define _PAGE_GLOBAL (_ULCAST_(1) << _PAGE_GLOBAL_SHIFT) #define _PAGE_HUGE (_ULCAST_(1) << _PAGE_HUGE_SHIFT) #define _PAGE_HGLOBAL (_ULCAST_(1) << _PAGE_HGLOBAL_SHIFT) +#ifdef CONFIG_32BIT +#define _PAGE_NO_READ 0 +#define _PAGE_NO_EXEC 0 +#define _PAGE_RPLV 0 +#else #define _PAGE_NO_READ (_ULCAST_(1) << _PAGE_NO_READ_SHIFT) #define _PAGE_NO_EXEC (_ULCAST_(1) << _PAGE_NO_EXEC_SHIFT) #define _PAGE_RPLV (_ULCAST_(1) << _PAGE_RPLV_SHIFT) +#endif #define _CACHE_MASK (_ULCAST_(3) << _CACHE_SHIFT) #define PFN_PTE_SHIFT (PAGE_SHIFT - 12 + _PAGE_PFN_SHIFT) diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index 03fb60432fde..f41a648a3d9e 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -11,6 +11,7 @@ #include <linux/compiler.h> #include <asm/addrspace.h> +#include <asm/asm.h> #include <asm/page.h> #include <asm/pgtable-bits.h> @@ -23,37 +24,45 @@ #endif #if CONFIG_PGTABLE_LEVELS == 2 -#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - 3)) +#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - PTRLOG)) #elif CONFIG_PGTABLE_LEVELS == 3 -#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - 3)) +#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - PTRLOG)) #define PMD_SIZE (1UL << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE-1)) -#define PGDIR_SHIFT (PMD_SHIFT + (PAGE_SHIFT - 3)) +#define PGDIR_SHIFT (PMD_SHIFT + (PAGE_SHIFT - PTRLOG)) #elif CONFIG_PGTABLE_LEVELS == 4 -#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - 3)) +#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - PTRLOG)) #define PMD_SIZE (1UL << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE-1)) -#define PUD_SHIFT (PMD_SHIFT + (PAGE_SHIFT - 3)) +#define PUD_SHIFT (PMD_SHIFT + (PAGE_SHIFT - PTRLOG)) #define PUD_SIZE (1UL << PUD_SHIFT) #define PUD_MASK (~(PUD_SIZE-1)) -#define PGDIR_SHIFT (PUD_SHIFT + (PAGE_SHIFT - 3)) +#define PGDIR_SHIFT (PUD_SHIFT + (PAGE_SHIFT - PTRLOG)) #endif #define PGDIR_SIZE (1UL << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) -#define VA_BITS (PGDIR_SHIFT + (PAGE_SHIFT - 3)) +#ifdef CONFIG_32BIT +#define VA_BITS 32 +#else +#define VA_BITS (PGDIR_SHIFT + (PAGE_SHIFT - PTRLOG)) +#endif -#define PTRS_PER_PGD (PAGE_SIZE >> 3) +#define PTRS_PER_PGD (PAGE_SIZE >> PTRLOG) #if CONFIG_PGTABLE_LEVELS > 3 -#define PTRS_PER_PUD (PAGE_SIZE >> 3) +#define PTRS_PER_PUD (PAGE_SIZE >> PTRLOG) #endif #if CONFIG_PGTABLE_LEVELS > 2 -#define PTRS_PER_PMD (PAGE_SIZE >> 3) +#define PTRS_PER_PMD (PAGE_SIZE >> PTRLOG) #endif -#define PTRS_PER_PTE (PAGE_SIZE >> 3) +#define PTRS_PER_PTE (PAGE_SIZE >> PTRLOG) +#ifdef CONFIG_32BIT +#define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) +#else #define USER_PTRS_PER_PGD ((TASK_SIZE64 / PGDIR_SIZE)?(TASK_SIZE64 / PGDIR_SIZE):1) +#endif #ifndef __ASSEMBLER__ @@ -74,11 +83,15 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; #define ZERO_PAGE(vaddr) virt_to_page(empty_zero_page) -/* - * TLB refill handlers may also map the vmalloc area into xkvrange. - * Avoid the first couple of pages so NULL pointer dereferences will - * still reliably trap. - */ +#ifdef CONFIG_32BIT + +#define VMALLOC_START (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE)) +#define VMALLOC_END (FIXADDR_START - (2 * PAGE_SIZE)) + +#endif + +#ifdef CONFIG_64BIT + #define MODULES_VADDR (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE)) #define MODULES_END (MODULES_VADDR + SZ_256M) @@ -106,6 +119,8 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; #define KFENCE_AREA_START (VMEMMAP_END + 1) #define KFENCE_AREA_END (KFENCE_AREA_START + KFENCE_AREA_SIZE - 1) +#endif + #define ptep_get(ptep) READ_ONCE(*(ptep)) #define pmdp_get(pmdp) READ_ONCE(*(pmdp)) @@ -277,7 +292,16 @@ extern void kernel_pte_init(void *addr); * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that * are !pte_none() && !pte_present(). * - * Format of swap PTEs: + * Format of 32bit swap PTEs: + * + * 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 + * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 + * <------------ offset -------------> E <- type -> <-- zeroes --> + * + * E is the exclusive marker that is not stored in swap entries. + * The zero'ed bits include _PAGE_PRESENT. + * + * Format of 64bit swap PTEs: * * 6 6 6 6 5 5 5 5 5 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3 3 * 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 @@ -290,16 +314,27 @@ extern void kernel_pte_init(void *addr); * E is the exclusive marker that is not stored in swap entries. * The zero'ed bits include _PAGE_PRESENT and _PAGE_PROTNONE. */ + +#define __SWP_TYPE_BITS (IS_ENABLED(CONFIG_32BIT) ? 5 : 7) +#define __SWP_TYPE_MASK ((1UL << __SWP_TYPE_BITS) - 1) +#define __SWP_TYPE_SHIFT (IS_ENABLED(CONFIG_32BIT) ? 8 : 16) +#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT + 1) + static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) -{ pte_t pte; pte_val(pte) = ((type & 0x7f) << 16) | (offset << 24); return pte; } +{ + pte_t pte; + pte_val(pte) = ((type & __SWP_TYPE_MASK) << __SWP_TYPE_SHIFT) | (offset << __SWP_OFFSET_SHIFT); + return pte; +} -#define __swp_type(x) (((x).val >> 16) & 0x7f) -#define __swp_offset(x) ((x).val >> 24) +#define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK) +#define __swp_offset(x) ((x).val >> __SWP_OFFSET_SHIFT) #define __swp_entry(type, offset) ((swp_entry_t) { pte_val(mk_swap_pte((type), (offset))) }) + +#define __swp_entry_to_pte(x) __pte((x).val) +#define __swp_entry_to_pmd(x) __pmd((x).val | _PAGE_HUGE) #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) -#define __swp_entry_to_pte(x) ((pte_t) { (x).val }) #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) -#define __swp_entry_to_pmd(x) ((pmd_t) { (x).val | _PAGE_HUGE }) static inline bool pte_swp_exclusive(pte_t pte) { diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h index 5cb568a60cf8..ecc8e50fffa8 100644 --- a/arch/loongarch/include/asm/stackframe.h +++ b/arch/loongarch/include/asm/stackframe.h @@ -38,22 +38,42 @@ cfi_restore \reg \offset \docfi .endm + .macro SETUP_TWINS temp + pcaddi t0, 0 + PTR_LI t1, ~TO_PHYS_MASK + and t0, t0, t1 + ori t0, t0, (1 << 4 | 1) + csrwr t0, LOONGARCH_CSR_DMWIN0 + PTR_LI t0, CSR_DMW1_INIT + csrwr t0, LOONGARCH_CSR_DMWIN1 + .endm + + .macro SETUP_MODES temp + /* Enable PG */ + li.w \temp, 0xb0 # PLV=0, IE=0, PG=1 + csrwr \temp, LOONGARCH_CSR_CRMD + li.w \temp, 0x04 # PLV=0, PIE=1, PWE=0 + csrwr \temp, LOONGARCH_CSR_PRMD + li.w \temp, 0x00 # FPE=0, SXE=0, ASXE=0, BTE=0 + csrwr \temp, LOONGARCH_CSR_EUEN + .endm + .macro SETUP_DMWINS temp - li.d \temp, CSR_DMW0_INIT # WUC, PLV0, 0x8000 xxxx xxxx xxxx + PTR_LI \temp, CSR_DMW0_INIT # SUC, PLV0, LA32: 0x8xxx xxxx, LA64: 0x8000 xxxx xxxx xxxx csrwr \temp, LOONGARCH_CSR_DMWIN0 - li.d \temp, CSR_DMW1_INIT # CAC, PLV0, 0x9000 xxxx xxxx xxxx + PTR_LI \temp, CSR_DMW1_INIT # CAC, PLV0, LA32: 0xaxxx xxxx, LA64: 0x9000 xxxx xxxx xxxx csrwr \temp, LOONGARCH_CSR_DMWIN1 - li.d \temp, CSR_DMW2_INIT # WUC, PLV0, 0xa000 xxxx xxxx xxxx + PTR_LI \temp, CSR_DMW2_INIT # WUC, PLV0, LA32: unavailable, LA64: 0xa000 xxxx xxxx xxxx csrwr \temp, LOONGARCH_CSR_DMWIN2 - li.d \temp, CSR_DMW3_INIT # 0x0, unused + PTR_LI \temp, CSR_DMW3_INIT # 0x0, unused csrwr \temp, LOONGARCH_CSR_DMWIN3 .endm /* Jump to the runtime virtual address. */ .macro JUMP_VIRT_ADDR temp1 temp2 - li.d \temp1, CACHE_BASE + PTR_LI \temp1, CACHE_BASE pcaddi \temp2, 0 - bstrins.d \temp1, \temp2, (DMW_PABITS - 1), 0 + PTR_BSTRINS \temp1, \temp2, (DMW_PABITS - 1), 0 jirl zero, \temp1, 0xc .endm @@ -171,7 +191,7 @@ andi t0, t0, 0x3 /* extract pplv bit */ beqz t0, 9f - li.d tp, ~_THREAD_MASK + LONG_LI tp, ~_THREAD_MASK and tp, tp, sp cfi_st u0, PT_R21, \docfi csrrd u0, PERCPU_BASE_KS diff --git a/arch/loongarch/include/asm/string.h b/arch/loongarch/include/asm/string.h index 5bb5a90d2681..bfa3fd879c7f 100644 --- a/arch/loongarch/include/asm/string.h +++ b/arch/loongarch/include/asm/string.h @@ -5,6 +5,7 @@ #ifndef _ASM_STRING_H #define _ASM_STRING_H +#ifdef CONFIG_64BIT #define __HAVE_ARCH_MEMSET extern void *memset(void *__s, int __c, size_t __count); extern void *__memset(void *__s, int __c, size_t __count); @@ -16,6 +17,7 @@ extern void *__memcpy(void *__to, __const__ void *__from, size_t __n); #define __HAVE_ARCH_MEMMOVE extern void *memmove(void *__dest, __const__ void *__src, size_t __n); extern void *__memmove(void *__dest, __const__ void *__src, size_t __n); +#endif #if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) diff --git a/arch/loongarch/include/asm/timex.h b/arch/loongarch/include/asm/timex.h index fb41e9e7a222..9ea52fad9690 100644 --- a/arch/loongarch/include/asm/timex.h +++ b/arch/loongarch/include/asm/timex.h @@ -18,7 +18,38 @@ typedef unsigned long cycles_t; static inline cycles_t get_cycles(void) { - return drdtime(); +#ifdef CONFIG_32BIT + return rdtime_l(); +#else + return rdtime_d(); +#endif +} + +#ifdef CONFIG_32BIT + +#define get_cycles_hi get_cycles_hi + +static inline cycles_t get_cycles_hi(void) +{ + return rdtime_h(); +} + +#endif + +static inline u64 get_cycles64(void) +{ +#ifdef CONFIG_32BIT + u32 hi, lo; + + do { + hi = rdtime_h(); + lo = rdtime_l(); + } while (hi != rdtime_h()); + + return ((u64)hi << 32) | lo; +#else + return rdtime_d(); +#endif } #endif /* __KERNEL__ */ diff --git a/arch/loongarch/include/asm/uaccess.h b/arch/loongarch/include/asm/uaccess.h index 0d22991ae430..4e259d490e45 100644 --- a/arch/loongarch/include/asm/uaccess.h +++ b/arch/loongarch/include/asm/uaccess.h @@ -19,10 +19,16 @@ #include <asm/asm-extable.h> #include <asm-generic/access_ok.h> +#define __LSW 0 +#define __MSW 1 + extern u64 __ua_limit; -#define __UA_ADDR ".dword" +#ifdef CONFIG_64BIT #define __UA_LIMIT __ua_limit +#else +#define __UA_LIMIT 0x80000000UL +#endif /* * get_user: - Get a simple variable from user space. @@ -126,6 +132,7 @@ extern u64 __ua_limit; * * Returns zero on success, or -EFAULT on error. */ + #define __put_user(x, ptr) \ ({ \ int __pu_err = 0; \ @@ -146,7 +153,7 @@ do { \ case 1: __get_data_asm(val, "ld.b", ptr); break; \ case 2: __get_data_asm(val, "ld.h", ptr); break; \ case 4: __get_data_asm(val, "ld.w", ptr); break; \ - case 8: __get_data_asm(val, "ld.d", ptr); break; \ + case 8: __get_data_asm_8(val, ptr); break; \ default: BUILD_BUG(); break; \ } \ } while (0) @@ -167,13 +174,39 @@ do { \ (val) = (__typeof__(*(ptr))) __gu_tmp; \ } +#ifdef CONFIG_64BIT +#define __get_data_asm_8(val, ptr) \ + __get_data_asm(val, "ld.d", ptr) +#else /* !CONFIG_64BIT */ +#define __get_data_asm_8(val, ptr) \ +{ \ + u32 __lo, __hi; \ + u32 __user *__ptr = (u32 __user *)(ptr); \ + \ + __asm__ __volatile__ ( \ + "1:\n" \ + " ld.w %1, %3 \n" \ + "2:\n" \ + " ld.w %2, %4 \n" \ + "3:\n" \ + _ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 3b, %0, %1) \ + _ASM_EXTABLE_UACCESS_ERR_ZERO(2b, 3b, %0, %1) \ + : "+r" (__gu_err), "=&r" (__lo), "=r" (__hi) \ + : "m" (__ptr[__LSW]), "m" (__ptr[__MSW])); \ + if (__gu_err) \ + __hi = 0; \ + (val) = (__typeof__(val))((__typeof__((val)-(val))) \ + ((((u64)__hi << 32) | __lo))); \ +} +#endif /* CONFIG_64BIT */ + #define __put_user_common(ptr, size) \ do { \ switch (size) { \ case 1: __put_data_asm("st.b", ptr); break; \ case 2: __put_data_asm("st.h", ptr); break; \ case 4: __put_data_asm("st.w", ptr); break; \ - case 8: __put_data_asm("st.d", ptr); break; \ + case 8: __put_data_asm_8(ptr); break; \ default: BUILD_BUG(); break; \ } \ } while (0) @@ -190,6 +223,30 @@ do { \ : "Jr" (__pu_val)); \ } +#ifdef CONFIG_64BIT +#define __put_data_asm_8(ptr) \ + __put_data_asm("st.d", ptr) +#else /* !CONFIG_64BIT */ +#define __put_data_asm_8(ptr) \ +{ \ + u32 __user *__ptr = (u32 __user *)(ptr); \ + u64 __x = (__typeof__((__pu_val)-(__pu_val)))(__pu_val); \ + \ + __asm__ __volatile__ ( \ + "1:\n" \ + " st.w %z3, %1 \n" \ + "2:\n" \ + " st.w %z4, %2 \n" \ + "3:\n" \ + _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %0) \ + _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %0) \ + : "+r" (__pu_err), \ + "=m" (__ptr[__LSW]), \ + "=m" (__ptr[__MSW]) \ + : "rJ" (__x), "rJ" (__x >> 32)); \ +} +#endif /* CONFIG_64BIT */ + #define __get_kernel_nofault(dst, src, type, err_label) \ do { \ int __gu_err = 0; \ diff --git a/arch/loongarch/include/asm/vdso/gettimeofday.h b/arch/loongarch/include/asm/vdso/gettimeofday.h index dcafabca9bb6..bae76767c693 100644 --- a/arch/loongarch/include/asm/vdso/gettimeofday.h +++ b/arch/loongarch/include/asm/vdso/gettimeofday.h @@ -12,6 +12,8 @@ #include <asm/unistd.h> #include <asm/vdso/vdso.h> +#ifdef CONFIG_GENERIC_GETTIMEOFDAY + #define VDSO_HAS_CLOCK_GETRES 1 static __always_inline long gettimeofday_fallback( @@ -89,6 +91,8 @@ static inline bool loongarch_vdso_hres_capable(void) } #define __arch_vdso_hres_capable loongarch_vdso_hres_capable +#endif /* CONFIG_GENERIC_GETTIMEOFDAY */ + #endif /* !__ASSEMBLER__ */ #endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/loongarch/include/uapi/asm/Kbuild b/arch/loongarch/include/uapi/asm/Kbuild index 517761419999..89ac01faa5ae 100644 --- a/arch/loongarch/include/uapi/asm/Kbuild +++ b/arch/loongarch/include/uapi/asm/Kbuild @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 +syscall-y += unistd_32.h syscall-y += unistd_64.h diff --git a/arch/loongarch/include/uapi/asm/ptrace.h b/arch/loongarch/include/uapi/asm/ptrace.h index 215e0f9e8aa3..b35c794323bc 100644 --- a/arch/loongarch/include/uapi/asm/ptrace.h +++ b/arch/loongarch/include/uapi/asm/ptrace.h @@ -61,8 +61,13 @@ struct user_lbt_state { struct user_watch_state { __u64 dbg_info; struct { +#if __BITS_PER_LONG == 32 + __u32 addr; + __u32 mask; +#else __u64 addr; __u64 mask; +#endif __u32 ctrl; __u32 pad; } dbg_regs[8]; @@ -71,8 +76,13 @@ struct user_watch_state { struct user_watch_state_v2 { __u64 dbg_info; struct { +#if __BITS_PER_LONG == 32 + __u32 addr; + __u32 mask; +#else __u64 addr; __u64 mask; +#endif __u32 ctrl; __u32 pad; } dbg_regs[14]; diff --git a/arch/loongarch/include/uapi/asm/unistd.h b/arch/loongarch/include/uapi/asm/unistd.h index 1f01980f9c94..e19c7f2f9f87 100644 --- a/arch/loongarch/include/uapi/asm/unistd.h +++ b/arch/loongarch/include/uapi/asm/unistd.h @@ -1,3 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#include <asm/bitsperlong.h> + +#if __BITS_PER_LONG == 32 +#include <asm/unistd_32.h> +#else #include <asm/unistd_64.h> +#endif |