diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-01-02 11:33:33 -0800 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-01-02 11:33:33 -0800 |
| commit | b993744a972722b4e15b8d2afee1a053767f4dd4 (patch) | |
| tree | 31b22617308623934bcbb617980b61b11f0e219c | |
| parent | 9b043680446067358913edc2e9dd71bf8ffae208 (diff) | |
| parent | bb85d206be208bbf834883e948125a35ac59993a (diff) | |
Merge tag 'loongarch-fixes-6.19-1' of git://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson
Pull LoongArch fixes from Huacai Chen:
"Complete CPUCFG registers definition, set correct protection_map[] for
VM_NONE/VM_SHARED, fix some bugs in the orc stack unwinder, ftrace and
BPF JIT"
* tag 'loongarch-fixes-6.19-1' of git://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson:
samples/ftrace: Adjust LoongArch register restore order in direct calls
LoongArch: BPF: Enhance the bpf_arch_text_poke() function
LoongArch: BPF: Enable trampoline-based tracing for module functions
LoongArch: BPF: Adjust the jump offset of tail calls
LoongArch: BPF: Save return address register ra to t0 before trampoline
LoongArch: BPF: Zero-extend bpf_tail_call() index
LoongArch: BPF: Sign extend kfunc call arguments
LoongArch: Refactor register restoration in ftrace_common_return
LoongArch: Enable exception fixup for specific ADE subcode
LoongArch: Remove unnecessary checks for ORC unwinder
LoongArch: Remove is_entry_func() and kernel_entry_end
LoongArch: Use UNWIND_HINT_END_OF_STACK for entry points
LoongArch: Set correct protection_map[] for VM_NONE/VM_SHARED
LoongArch: Complete CPUCFG registers definition
| -rw-r--r-- | arch/loongarch/include/asm/loongarch.h | 7 | ||||
| -rw-r--r-- | arch/loongarch/kernel/head.S | 4 | ||||
| -rw-r--r-- | arch/loongarch/kernel/mcount_dyn.S | 14 | ||||
| -rw-r--r-- | arch/loongarch/kernel/traps.c | 5 | ||||
| -rw-r--r-- | arch/loongarch/kernel/unwind_orc.c | 27 | ||||
| -rw-r--r-- | arch/loongarch/mm/cache.c | 8 | ||||
| -rw-r--r-- | arch/loongarch/net/bpf_jit.c | 58 | ||||
| -rw-r--r-- | arch/loongarch/net/bpf_jit.h | 26 | ||||
| -rw-r--r-- | samples/ftrace/ftrace-direct-modify.c | 8 | ||||
| -rw-r--r-- | samples/ftrace/ftrace-direct-multi-modify.c | 8 | ||||
| -rw-r--r-- | samples/ftrace/ftrace-direct-multi.c | 4 | ||||
| -rw-r--r-- | samples/ftrace/ftrace-direct-too.c | 4 | ||||
| -rw-r--r-- | samples/ftrace/ftrace-direct.c | 4 |
13 files changed, 120 insertions, 57 deletions
diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index e6b8ff61c8cc..553c4dc7a156 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -94,6 +94,12 @@ #define CPUCFG2_LSPW BIT(21) #define CPUCFG2_LAM BIT(22) #define CPUCFG2_PTW BIT(24) +#define CPUCFG2_FRECIPE BIT(25) +#define CPUCFG2_DIV32 BIT(26) +#define CPUCFG2_LAM_BH BIT(27) +#define CPUCFG2_LAMCAS BIT(28) +#define CPUCFG2_LLACQ_SCREL BIT(29) +#define CPUCFG2_SCQ BIT(30) #define LOONGARCH_CPUCFG3 0x3 #define CPUCFG3_CCDMA BIT(0) @@ -108,6 +114,7 @@ #define CPUCFG3_SPW_HG_HF BIT(11) #define CPUCFG3_RVA BIT(12) #define CPUCFG3_RVAMAX GENMASK(16, 13) +#define CPUCFG3_DBAR_HINTS BIT(17) #define CPUCFG3_ALDORDER_CAP BIT(18) /* All address load ordered, capability */ #define CPUCFG3_ASTORDER_CAP BIT(19) /* All address store ordered, capability */ #define CPUCFG3_ALDORDER_STA BIT(20) /* All address load ordered, status */ diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index aba548db2446..7f288e89573b 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -42,6 +42,7 @@ SYM_DATA(kernel_fsize, .long _kernel_fsize); .align 12 SYM_CODE_START(kernel_entry) # kernel entry point + UNWIND_HINT_END_OF_STACK SETUP_TWINS SETUP_MODES t0 @@ -113,6 +114,7 @@ SYM_CODE_END(kernel_entry) * function after setting up the stack and tp registers. */ SYM_CODE_START(smpboot_entry) + UNWIND_HINT_END_OF_STACK SETUP_TWINS SETUP_MODES t0 @@ -142,5 +144,3 @@ SYM_CODE_START(smpboot_entry) SYM_CODE_END(smpboot_entry) #endif /* CONFIG_SMP */ - -SYM_ENTRY(kernel_entry_end, SYM_L_GLOBAL, SYM_A_NONE) diff --git a/arch/loongarch/kernel/mcount_dyn.S b/arch/loongarch/kernel/mcount_dyn.S index d6b474ad1d5e..5729c20e5b8b 100644 --- a/arch/loongarch/kernel/mcount_dyn.S +++ b/arch/loongarch/kernel/mcount_dyn.S @@ -94,7 +94,6 @@ SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) * at the callsite, so there is no need to restore the T series regs. */ ftrace_common_return: - PTR_L ra, sp, PT_R1 PTR_L a0, sp, PT_R4 PTR_L a1, sp, PT_R5 PTR_L a2, sp, PT_R6 @@ -104,12 +103,17 @@ ftrace_common_return: PTR_L a6, sp, PT_R10 PTR_L a7, sp, PT_R11 PTR_L fp, sp, PT_R22 - PTR_L t0, sp, PT_ERA PTR_L t1, sp, PT_R13 - PTR_ADDI sp, sp, PT_SIZE bnez t1, .Ldirect + + PTR_L ra, sp, PT_R1 + PTR_L t0, sp, PT_ERA + PTR_ADDI sp, sp, PT_SIZE jr t0 .Ldirect: + PTR_L t0, sp, PT_R1 + PTR_L ra, sp, PT_ERA + PTR_ADDI sp, sp, PT_SIZE jr t1 SYM_CODE_END(ftrace_common) @@ -161,6 +165,8 @@ SYM_CODE_END(return_to_handler) #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS SYM_CODE_START(ftrace_stub_direct_tramp) UNWIND_HINT_UNDEFINED - jr t0 + move t1, ra + move ra, t0 + jr t1 SYM_CODE_END(ftrace_stub_direct_tramp) #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c index 004b8ebf0051..5d49b742e3bf 100644 --- a/arch/loongarch/kernel/traps.c +++ b/arch/loongarch/kernel/traps.c @@ -535,10 +535,15 @@ out: asmlinkage void noinstr do_ade(struct pt_regs *regs) { irqentry_state_t state = irqentry_enter(regs); + unsigned int esubcode = FIELD_GET(CSR_ESTAT_ESUBCODE, regs->csr_estat); + + if ((esubcode == EXSUBCODE_ADEM) && fixup_exception(regs)) + goto out; die_if_kernel("Kernel ade access", regs); force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)regs->csr_badvaddr); +out: irqentry_exit(regs, state); } diff --git a/arch/loongarch/kernel/unwind_orc.c b/arch/loongarch/kernel/unwind_orc.c index 0d5fa64a2225..8a6e3429a860 100644 --- a/arch/loongarch/kernel/unwind_orc.c +++ b/arch/loongarch/kernel/unwind_orc.c @@ -348,24 +348,10 @@ void unwind_start(struct unwind_state *state, struct task_struct *task, } EXPORT_SYMBOL_GPL(unwind_start); -static bool is_entry_func(unsigned long addr) -{ - extern u32 kernel_entry; - extern u32 kernel_entry_end; - - return addr >= (unsigned long)&kernel_entry && addr < (unsigned long)&kernel_entry_end; -} - static inline unsigned long bt_address(unsigned long ra) { extern unsigned long eentry; - if (__kernel_text_address(ra)) - return ra; - - if (__module_text_address(ra)) - return ra; - if (ra >= eentry && ra < eentry + EXCCODE_INT_END * VECSIZE) { unsigned long func; unsigned long type = (ra - eentry) / VECSIZE; @@ -383,10 +369,13 @@ static inline unsigned long bt_address(unsigned long ra) break; } - return func + offset; + ra = func + offset; } - return ra; + if (__kernel_text_address(ra)) + return ra; + + return 0; } bool unwind_next_frame(struct unwind_state *state) @@ -402,9 +391,6 @@ bool unwind_next_frame(struct unwind_state *state) /* Don't let modules unload while we're reading their ORC data. */ guard(rcu)(); - if (is_entry_func(state->pc)) - goto end; - orc = orc_find(state->pc); if (!orc) { /* @@ -512,9 +498,6 @@ bool unwind_next_frame(struct unwind_state *state) goto err; } - if (!__kernel_text_address(state->pc)) - goto err; - return true; err: diff --git a/arch/loongarch/mm/cache.c b/arch/loongarch/mm/cache.c index 6be04d36ca07..496916845ff7 100644 --- a/arch/loongarch/mm/cache.c +++ b/arch/loongarch/mm/cache.c @@ -160,8 +160,8 @@ void cpu_cache_init(void) static const pgprot_t protection_map[16] = { [VM_NONE] = __pgprot(_CACHE_CC | _PAGE_USER | - _PAGE_PROTNONE | _PAGE_NO_EXEC | - _PAGE_NO_READ), + _PAGE_NO_EXEC | _PAGE_NO_READ | + (_PAGE_PROTNONE ? : _PAGE_PRESENT)), [VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC), @@ -180,8 +180,8 @@ static const pgprot_t protection_map[16] = { [VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT), [VM_SHARED] = __pgprot(_CACHE_CC | _PAGE_USER | - _PAGE_PROTNONE | _PAGE_NO_EXEC | - _PAGE_NO_READ), + _PAGE_NO_EXEC | _PAGE_NO_READ | + (_PAGE_PROTNONE ? : _PAGE_PRESENT)), [VM_SHARED | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | _PAGE_USER | _PAGE_PRESENT | _PAGE_NO_EXEC), diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 8dc58781b8eb..d1d5a65308b9 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -139,6 +139,7 @@ static void build_prologue(struct jit_ctx *ctx) stack_adjust = round_up(stack_adjust, 16); stack_adjust += bpf_stack_adjust; + move_reg(ctx, LOONGARCH_GPR_T0, LOONGARCH_GPR_RA); /* Reserve space for the move_imm + jirl instruction */ for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) emit_insn(ctx, nop); @@ -238,7 +239,7 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) * Call the next bpf prog and skip the first instruction * of TCC initialization. */ - emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 6); + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 7); } } @@ -280,6 +281,8 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) * goto out; */ tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0]; + emit_zext_32(ctx, a2, true); + off = offsetof(struct bpf_array, map.max_entries); emit_insn(ctx, ldwu, t1, a1, off); /* bgeu $a2, $t1, jmp_offset */ @@ -950,6 +953,22 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); } + if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { + const struct btf_func_model *m; + int i; + + m = bpf_jit_find_kfunc_model(ctx->prog, insn); + if (!m) + return -EINVAL; + + for (i = 0; i < m->nr_args; i++) { + u8 reg = regmap[BPF_REG_1 + i]; + bool sign = m->arg_flags[i] & BTF_FMODEL_SIGNED_ARG; + + emit_abi_ext(ctx, reg, m->arg_size[i], sign); + } + } + move_addr(ctx, t1, func_addr); emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0); @@ -1265,7 +1284,7 @@ static int emit_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) return 0; } - return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_T0 : LOONGARCH_GPR_ZERO, (u64)target); + return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_RA : LOONGARCH_GPR_ZERO, (u64)target); } static int emit_call(struct jit_ctx *ctx, u64 addr) @@ -1290,15 +1309,30 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, { int ret; bool is_call; + unsigned long size = 0; + unsigned long offset = 0; + void *image = NULL; + char namebuf[KSYM_NAME_LEN]; u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; /* Only poking bpf text is supported. Since kernel function entry * is set up by ftrace, we rely on ftrace to poke kernel functions. */ - if (!is_bpf_text_address((unsigned long)ip)) + if (!__bpf_address_lookup((unsigned long)ip, &size, &offset, namebuf)) return -ENOTSUPP; + image = ip - offset; + + /* zero offset means we're poking bpf prog entry */ + if (offset == 0) { + /* skip to the nop instruction in bpf prog entry: + * move t0, ra + * nop + */ + ip = image + LOONGARCH_INSN_SIZE; + } + is_call = old_t == BPF_MOD_CALL; ret = emit_jump_or_nops(old_addr, ip, old_insns, is_call); if (ret) @@ -1622,14 +1656,12 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i /* To traced function */ /* Ftrace jump skips 2 NOP instructions */ - if (is_kernel_text((unsigned long)orig_call)) + if (is_kernel_text((unsigned long)orig_call) || + is_module_text_address((unsigned long)orig_call)) orig_call += LOONGARCH_FENTRY_NBYTES; /* Direct jump skips 5 NOP instructions */ else if (is_bpf_text_address((unsigned long)orig_call)) orig_call += LOONGARCH_BPF_FENTRY_NBYTES; - /* Module tracing not supported - cause kernel lockups */ - else if (is_module_text_address((unsigned long)orig_call)) - return -ENOTSUPP; if (flags & BPF_TRAMP_F_CALL_ORIG) { move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im); @@ -1722,12 +1754,16 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16); - if (flags & BPF_TRAMP_F_SKIP_FRAME) + if (flags & BPF_TRAMP_F_SKIP_FRAME) { /* return to parent function */ - emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); - else - /* return to traced function */ + move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0); emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0); + } else { + /* return to traced function */ + move_reg(ctx, LOONGARCH_GPR_T1, LOONGARCH_GPR_RA); + move_reg(ctx, LOONGARCH_GPR_RA, LOONGARCH_GPR_T0); + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T1, 0); + } } ret = ctx->idx; diff --git a/arch/loongarch/net/bpf_jit.h b/arch/loongarch/net/bpf_jit.h index 5697158fd164..75b6330030a9 100644 --- a/arch/loongarch/net/bpf_jit.h +++ b/arch/loongarch/net/bpf_jit.h @@ -88,6 +88,32 @@ static inline void emit_sext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, boo emit_insn(ctx, addiw, reg, reg, 0); } +/* Emit proper extension according to ABI requirements. + * Note that it requires a value of size `size` already resides in register `reg`. + */ +static inline void emit_abi_ext(struct jit_ctx *ctx, int reg, u8 size, bool sign) +{ + /* ABI requires unsigned char/short to be zero-extended */ + if (!sign && (size == 1 || size == 2)) + return; + + switch (size) { + case 1: + emit_insn(ctx, extwb, reg, reg); + break; + case 2: + emit_insn(ctx, extwh, reg, reg); + break; + case 4: + emit_insn(ctx, addiw, reg, reg, 0); + break; + case 8: + break; + default: + pr_warn("bpf_jit: invalid size %d for extension\n", size); + } +} + static inline void move_addr(struct jit_ctx *ctx, enum loongarch_gpr rd, u64 addr) { u64 imm_11_0, imm_31_12, imm_51_32, imm_63_52; diff --git a/samples/ftrace/ftrace-direct-modify.c b/samples/ftrace/ftrace-direct-modify.c index da3a9f2091f5..1ba1927b548e 100644 --- a/samples/ftrace/ftrace-direct-modify.c +++ b/samples/ftrace/ftrace-direct-modify.c @@ -176,8 +176,8 @@ asm ( " st.d $t0, $sp, 0\n" " st.d $ra, $sp, 8\n" " bl my_direct_func1\n" -" ld.d $t0, $sp, 0\n" -" ld.d $ra, $sp, 8\n" +" ld.d $ra, $sp, 0\n" +" ld.d $t0, $sp, 8\n" " addi.d $sp, $sp, 16\n" " jr $t0\n" " .size my_tramp1, .-my_tramp1\n" @@ -189,8 +189,8 @@ asm ( " st.d $t0, $sp, 0\n" " st.d $ra, $sp, 8\n" " bl my_direct_func2\n" -" ld.d $t0, $sp, 0\n" -" ld.d $ra, $sp, 8\n" +" ld.d $ra, $sp, 0\n" +" ld.d $t0, $sp, 8\n" " addi.d $sp, $sp, 16\n" " jr $t0\n" " .size my_tramp2, .-my_tramp2\n" diff --git a/samples/ftrace/ftrace-direct-multi-modify.c b/samples/ftrace/ftrace-direct-multi-modify.c index 8f7986d698d8..7a7822dfeb50 100644 --- a/samples/ftrace/ftrace-direct-multi-modify.c +++ b/samples/ftrace/ftrace-direct-multi-modify.c @@ -199,8 +199,8 @@ asm ( " move $a0, $t0\n" " bl my_direct_func1\n" " ld.d $a0, $sp, 0\n" -" ld.d $t0, $sp, 8\n" -" ld.d $ra, $sp, 16\n" +" ld.d $ra, $sp, 8\n" +" ld.d $t0, $sp, 16\n" " addi.d $sp, $sp, 32\n" " jr $t0\n" " .size my_tramp1, .-my_tramp1\n" @@ -215,8 +215,8 @@ asm ( " move $a0, $t0\n" " bl my_direct_func2\n" " ld.d $a0, $sp, 0\n" -" ld.d $t0, $sp, 8\n" -" ld.d $ra, $sp, 16\n" +" ld.d $ra, $sp, 8\n" +" ld.d $t0, $sp, 16\n" " addi.d $sp, $sp, 32\n" " jr $t0\n" " .size my_tramp2, .-my_tramp2\n" diff --git a/samples/ftrace/ftrace-direct-multi.c b/samples/ftrace/ftrace-direct-multi.c index db326c81a27d..3fe6ddaf0b69 100644 --- a/samples/ftrace/ftrace-direct-multi.c +++ b/samples/ftrace/ftrace-direct-multi.c @@ -131,8 +131,8 @@ asm ( " move $a0, $t0\n" " bl my_direct_func\n" " ld.d $a0, $sp, 0\n" -" ld.d $t0, $sp, 8\n" -" ld.d $ra, $sp, 16\n" +" ld.d $ra, $sp, 8\n" +" ld.d $t0, $sp, 16\n" " addi.d $sp, $sp, 32\n" " jr $t0\n" " .size my_tramp, .-my_tramp\n" diff --git a/samples/ftrace/ftrace-direct-too.c b/samples/ftrace/ftrace-direct-too.c index 3d0fa260332d..bf2411aa6fd7 100644 --- a/samples/ftrace/ftrace-direct-too.c +++ b/samples/ftrace/ftrace-direct-too.c @@ -143,8 +143,8 @@ asm ( " ld.d $a0, $sp, 0\n" " ld.d $a1, $sp, 8\n" " ld.d $a2, $sp, 16\n" -" ld.d $t0, $sp, 24\n" -" ld.d $ra, $sp, 32\n" +" ld.d $ra, $sp, 24\n" +" ld.d $t0, $sp, 32\n" " addi.d $sp, $sp, 48\n" " jr $t0\n" " .size my_tramp, .-my_tramp\n" diff --git a/samples/ftrace/ftrace-direct.c b/samples/ftrace/ftrace-direct.c index 956834b0d19a..5368c8c39cbb 100644 --- a/samples/ftrace/ftrace-direct.c +++ b/samples/ftrace/ftrace-direct.c @@ -124,8 +124,8 @@ asm ( " st.d $ra, $sp, 16\n" " bl my_direct_func\n" " ld.d $a0, $sp, 0\n" -" ld.d $t0, $sp, 8\n" -" ld.d $ra, $sp, 16\n" +" ld.d $ra, $sp, 8\n" +" ld.d $t0, $sp, 16\n" " addi.d $sp, $sp, 32\n" " jr $t0\n" " .size my_tramp, .-my_tramp\n" |
