From 0936243cabf0caf46f1a42606325ab93cfa05a6a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 18 Apr 2023 16:36:03 +0200 Subject: arm64: entry: Preserve/restore X29 even for compat tasks Currently, the KPTI trampoline code for returning to user space takes care to only preserve X29 into FAR_EL1 for native tasks, as compat tasks don't have access to this register anyway, and so preserving it is not necessary. It also means it does not need to be restored, and so we have two code paths for returning back to user space: the native one that restores X29 from FAR_EL1, and the compat one that leaves X29 clobbered, containing the value of TTBR1_EL1, which carries a physical address pointing somewhere into the kernel image. This is needlessly complex, and given that FAR_EL1 becomes UNKNOWN after an exception return anway, the only benefit of avoiding the preserve and restore is that we can skip the system register write and read. So let's simplify this, and collapse the two code paths into one that always preserves X29 into FAR_EL1, and always restores it again after the TTBR switch. Signed-off-by: Ard Biesheuvel Reviewed-by: Will Deacon Link: https://lore.kernel.org/r/20230418143604.1176437-2-ardb@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/kernel/entry.S | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index ab2a6e33c052..16fbd0d9790d 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -435,13 +435,9 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 eret alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - bne 4f msr far_el1, x29 tramp_alias x30, tramp_exit_native, x29 br x30 -4: - tramp_alias x30, tramp_exit_compat, x29 - br x30 #endif .else ldr lr, [sp, #S_LR] @@ -740,9 +736,7 @@ alternative_else_nop_endif msr vbar_el1, x30 ldr lr, [sp, #S_LR] tramp_unmap_kernel x29 - .if \regsize == 64 mrs x29, far_el1 - .endif add sp, sp, #PT_REGS_SIZE // restore sp eret sb @@ -780,10 +774,6 @@ SYM_CODE_END(tramp_vectors) SYM_CODE_START(tramp_exit_native) tramp_exit SYM_CODE_END(tramp_exit_native) - -SYM_CODE_START(tramp_exit_compat) - tramp_exit 32 -SYM_CODE_END(tramp_exit_compat) .popsection // .entry.tramp.text #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ -- cgit From 211ceca377f40ff46aef8ceb6e26cf4e7efecaf1 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 18 Apr 2023 16:36:04 +0200 Subject: arm64: entry: Simplify tramp_alias macro and tramp_exit routine The tramp_alias macro constructs the virtual alias of a symbol in the trampoline text mapping, based on its kernel text address, and does so in a way that is more convoluted than necessary. So let's simplify that. Also, now that the address of the vector table is kept in a per-CPU variable, there is no need to defer the load and the assignment of VBAR_EL1 to tramp_exit(). This means we can use a PC-relative reference to the per-CPU variable instead of storing its absolute address in a global variable in the trampoline rodata. And given that tramp_alias no longer needs a temp register, this means we can restore X30 earlier as well, and only leave X29 for tramp_exit() to restore. While at it, give some related symbols static linkage, considering that they are only referenced from the object file that defines them. Signed-off-by: Ard Biesheuvel Reviewed-by: Will Deacon Link: https://lore.kernel.org/r/20230418143604.1176437-3-ardb@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/kernel/entry.S | 47 ++++++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 16fbd0d9790d..a40e5e50fa55 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -101,12 +101,11 @@ .org .Lventry_start\@ + 128 // Did we overflow the ventry slot? .endm - .macro tramp_alias, dst, sym, tmp - mov_q \dst, TRAMP_VALIAS - adr_l \tmp, \sym - add \dst, \dst, \tmp - adr_l \tmp, .entry.tramp.text - sub \dst, \dst, \tmp + .macro tramp_alias, dst, sym + .set .Lalias\@, TRAMP_VALIAS + \sym - .entry.tramp.text + movz \dst, :abs_g2_s:.Lalias\@ + movk \dst, :abs_g1_nc:.Lalias\@ + movk \dst, :abs_g0_nc:.Lalias\@ .endm /* @@ -436,8 +435,13 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 msr far_el1, x29 - tramp_alias x30, tramp_exit_native, x29 - br x30 + + ldr_this_cpu x30, this_cpu_vector, x29 + tramp_alias x29, tramp_exit + msr vbar_el1, x30 // install vector table + ldr lr, [sp, #S_LR] // restore x30 + add sp, sp, #PT_REGS_SIZE // restore sp + br x29 #endif .else ldr lr, [sp, #S_LR] @@ -728,20 +732,6 @@ alternative_else_nop_endif .org 1b + 128 // Did we overflow the ventry slot? .endm - .macro tramp_exit, regsize = 64 - tramp_data_read_var x30, this_cpu_vector - get_this_cpu_offset x29 - ldr x30, [x30, x29] - - msr vbar_el1, x30 - ldr lr, [sp, #S_LR] - tramp_unmap_kernel x29 - mrs x29, far_el1 - add sp, sp, #PT_REGS_SIZE // restore sp - eret - sb - .endm - .macro generate_tramp_vector, kpti, bhb .Lvector_start\@: .space 0x400 @@ -762,7 +752,7 @@ alternative_else_nop_endif */ .pushsection ".entry.tramp.text", "ax" .align 11 -SYM_CODE_START_NOALIGN(tramp_vectors) +SYM_CODE_START_LOCAL_NOALIGN(tramp_vectors) #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW @@ -771,9 +761,12 @@ SYM_CODE_START_NOALIGN(tramp_vectors) generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE SYM_CODE_END(tramp_vectors) -SYM_CODE_START(tramp_exit_native) - tramp_exit -SYM_CODE_END(tramp_exit_native) +SYM_CODE_START_LOCAL(tramp_exit) + tramp_unmap_kernel x29 + mrs x29, far_el1 // restore x29 + eret + sb +SYM_CODE_END(tramp_exit) .popsection // .entry.tramp.text #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */ @@ -1067,7 +1060,7 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0 alternative_else_nop_endif #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 - tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3 + tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline br x5 #endif SYM_CODE_END(__sdei_asm_handler) -- cgit From 320a93d4df48a378ebf923639fa62770676b80db Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:28 +0200 Subject: arm64: xor-neon: mark xor_arm64_neon_*() static The only references to these functions are in the same file, and there is no prototype, which causes a harmless warning: arch/arm64/lib/xor-neon.c:13:6: error: no previous prototype for 'xor_arm64_neon_2' [-Werror=missing-prototypes] arch/arm64/lib/xor-neon.c:40:6: error: no previous prototype for 'xor_arm64_neon_3' [-Werror=missing-prototypes] arch/arm64/lib/xor-neon.c:76:6: error: no previous prototype for 'xor_arm64_neon_4' [-Werror=missing-prototypes] arch/arm64/lib/xor-neon.c:121:6: error: no previous prototype for 'xor_arm64_neon_5' [-Werror=missing-prototypes] Fixes: cc9f8349cb33 ("arm64: crypto: add NEON accelerated XOR implementation") Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-2-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/lib/xor-neon.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/arm64/lib/xor-neon.c b/arch/arm64/lib/xor-neon.c index 96b171995d19..f9a53b7f9842 100644 --- a/arch/arm64/lib/xor-neon.c +++ b/arch/arm64/lib/xor-neon.c @@ -10,7 +10,7 @@ #include #include -void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1, +static void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1, const unsigned long * __restrict p2) { uint64_t *dp1 = (uint64_t *)p1; @@ -37,7 +37,7 @@ void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1, } while (--lines > 0); } -void xor_arm64_neon_3(unsigned long bytes, unsigned long * __restrict p1, +static void xor_arm64_neon_3(unsigned long bytes, unsigned long * __restrict p1, const unsigned long * __restrict p2, const unsigned long * __restrict p3) { @@ -73,7 +73,7 @@ void xor_arm64_neon_3(unsigned long bytes, unsigned long * __restrict p1, } while (--lines > 0); } -void xor_arm64_neon_4(unsigned long bytes, unsigned long * __restrict p1, +static void xor_arm64_neon_4(unsigned long bytes, unsigned long * __restrict p1, const unsigned long * __restrict p2, const unsigned long * __restrict p3, const unsigned long * __restrict p4) @@ -118,7 +118,7 @@ void xor_arm64_neon_4(unsigned long bytes, unsigned long * __restrict p1, } while (--lines > 0); } -void xor_arm64_neon_5(unsigned long bytes, unsigned long * __restrict p1, +static void xor_arm64_neon_5(unsigned long bytes, unsigned long * __restrict p1, const unsigned long * __restrict p2, const unsigned long * __restrict p3, const unsigned long * __restrict p4, -- cgit From aea197160d7426d876a8040f370373cf412e3ad6 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:29 +0200 Subject: arm64: add scs_patch_vmlinux prototype scs_patch_vmlinux() is only called from assembler code, so there is no prototype, but adding one avoids this warning: arch/arm64/kernel/patch-scs.c:254:24: error: no previous prototype for function 'scs_patch_vmlinux' [-Werror,-Wmissing-prototypes] Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-3-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/scs.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/include/asm/scs.h b/arch/arm64/include/asm/scs.h index 13df982a0808..3fdae5fe3142 100644 --- a/arch/arm64/include/asm/scs.h +++ b/arch/arm64/include/asm/scs.h @@ -73,6 +73,7 @@ static inline void dynamic_scs_init(void) {} #endif int scs_patch(const u8 eh_frame[], int size); +asmlinkage void scs_patch_vmlinux(void); #endif /* __ASSEMBLY __ */ -- cgit From 6ac19f96515e1f5198503701d3aecf60d5bc83e5 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:30 +0200 Subject: arm64: avoid prototype warnings for syscalls With W=1 warnings enabled, there are lots of complaints about missing prototypes for system calls, plus a few other related ones: arch/arm64/kernel/sys_compat.c:68:6: error: no previous prototype for 'compat_arm_syscall' [-Werror=missing-prototypes] arch/arm64/include/asm/syscall_wrapper.h:76:32: error: no previous prototype for '__arm64_sys_io_setup' [-Werror=missing-prototypes] arch/arm64/include/asm/syscall_wrapper.h:41:32: error: no previous prototype for '__arm64_compat_sys_io_setup' [-Werror=missing-prototypes] arch/arm64/include/asm/syscall_wrapper.h:76:32: error: no previous prototype for '__arm64_sys_io_destroy' [-Werror=missing-prototypes] arch/arm64/include/asm/syscall_wrapper.h:76:32: error: no previous prototype for '__arm64_sys_io_submit' [-Werror=missing-prototypes] Add declarations to the syscall macros to avoid all of these, plus one for the compat syscall entry. Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-4-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/compat.h | 2 ++ arch/arm64/include/asm/syscall_wrapper.h | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/arch/arm64/include/asm/compat.h b/arch/arm64/include/asm/compat.h index 74575c3d6987..ae904a1ad529 100644 --- a/arch/arm64/include/asm/compat.h +++ b/arch/arm64/include/asm/compat.h @@ -96,6 +96,8 @@ static inline int is_compat_thread(struct thread_info *thread) return test_ti_thread_flag(thread, TIF_32BIT); } +long compat_arm_syscall(struct pt_regs *regs, int scno); + #else /* !CONFIG_COMPAT */ static inline int is_compat_thread(struct thread_info *thread) diff --git a/arch/arm64/include/asm/syscall_wrapper.h b/arch/arm64/include/asm/syscall_wrapper.h index d30217c21eff..17f687510c48 100644 --- a/arch/arm64/include/asm/syscall_wrapper.h +++ b/arch/arm64/include/asm/syscall_wrapper.h @@ -38,6 +38,7 @@ asmlinkage long __arm64_compat_sys_##sname(const struct pt_regs *__unused) #define COND_SYSCALL_COMPAT(name) \ + asmlinkage long __arm64_compat_sys_##name(const struct pt_regs *regs); \ asmlinkage long __weak __arm64_compat_sys_##name(const struct pt_regs *regs) \ { \ return sys_ni_syscall(); \ @@ -53,6 +54,7 @@ ALLOW_ERROR_INJECTION(__arm64_sys##name, ERRNO); \ static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)); \ static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)); \ + asmlinkage long __arm64_sys##name(const struct pt_regs *regs); \ asmlinkage long __arm64_sys##name(const struct pt_regs *regs) \ { \ return __se_sys##name(SC_ARM64_REGS_TO_ARGS(x,__VA_ARGS__)); \ @@ -73,11 +75,13 @@ asmlinkage long __arm64_sys_##sname(const struct pt_regs *__unused) #define COND_SYSCALL(name) \ + asmlinkage long __arm64_sys_##name(const struct pt_regs *regs); \ asmlinkage long __weak __arm64_sys_##name(const struct pt_regs *regs) \ { \ return sys_ni_syscall(); \ } +asmlinkage long __arm64_sys_ni_syscall(const struct pt_regs *__unused); #define SYS_NI(name) SYSCALL_ALIAS(__arm64_sys_##name, sys_ni_posix_timers); #endif /* __ASM_SYSCALL_WRAPPER_H */ -- cgit From ec3a3db7100ddebb32ed5b1052c1cd1136057230 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:31 +0200 Subject: arm64: move cpu_suspend_set_dbg_restorer() prototype to header The cpu_suspend_set_dbg_restorer() function is called by the hw_breakpoint code but defined in another file. Since the declaration is in the same file as the caller, the compiler warns about the definition without a prior prototype: arch/arm64/kernel/suspend.c:35:13: error: no previous prototype for 'cpu_suspend_set_dbg_restorer' [-Werror=missing-prototypes] Move it into the corresponding header instead to avoid the warning. Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-5-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/hw_breakpoint.h | 8 ++++++++ arch/arm64/kernel/hw_breakpoint.c | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h index fa4c6ff3aa9b..84055329cd8b 100644 --- a/arch/arm64/include/asm/hw_breakpoint.h +++ b/arch/arm64/include/asm/hw_breakpoint.h @@ -154,4 +154,12 @@ static inline int get_num_wrps(void) ID_AA64DFR0_EL1_WRPs_SHIFT); } +#ifdef CONFIG_CPU_PM +extern void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int)); +#else +static inline void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int)) +{ +} +#endif + #endif /* __ASM_BREAKPOINT_H */ diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index b29a311bb055..db2a1861bb97 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -973,14 +973,6 @@ static int hw_breakpoint_reset(unsigned int cpu) return 0; } -#ifdef CONFIG_CPU_PM -extern void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int)); -#else -static inline void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int)) -{ -} -#endif - /* * One-time initialisation. */ -- cgit From 010089e9d3fe689e439fe3b78ed859a0782fbad0 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:32 +0200 Subject: arm64: spectre: provide prototypes for internal functions The helpers in proton-pack.c are called from assembler and have no prototype in a header, which causes a W=1 warning: arch/arm64/kernel/proton-pack.c:568:13: error: no previous prototype for 'spectre_v4_patch_fw_mitigation_enable' [-Werror=missing-prototypes] arch/arm64/kernel/proton-pack.c:588:13: error: no previous prototype for 'smccc_patch_fw_mitigation_conduit' [-Werror=missing-prototypes] arch/arm64/kernel/proton-pack.c:1064:14: error: no previous prototype for 'spectre_bhb_patch_loop_mitigation_enable' [-Werror=missing-prototypes] arch/arm64/kernel/proton-pack.c:1075:14: error: no previous prototype for 'spectre_bhb_patch_fw_mitigation_enabled' [-Werror=missing-prototypes] Add these to asm/spectre.h, which contains related declarations already. Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-6-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/spectre.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/arch/arm64/include/asm/spectre.h b/arch/arm64/include/asm/spectre.h index db7b371b367c..9cc501450486 100644 --- a/arch/arm64/include/asm/spectre.h +++ b/arch/arm64/include/asm/spectre.h @@ -100,5 +100,21 @@ bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, int sco u8 spectre_bhb_loop_affected(int scope); void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *__unused); bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr); + +void spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt, __le32 *origptr, + __le32 *updptr, int nr_inst); +void smccc_patch_fw_mitigation_conduit(struct alt_instr *alt, __le32 *origptr, + __le32 *updptr, int nr_inst); +void spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt, __le32 *origptr, + __le32 *updptr, int nr_inst); +void spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt, __le32 *origptr, + __le32 *updptr, int nr_inst); +void spectre_bhb_patch_loop_iter(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst); +void spectre_bhb_patch_wa3(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst); +void spectre_bhb_patch_clearbhb(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst); + #endif /* __ASSEMBLY__ */ #endif /* __ASM_SPECTRE_H */ -- cgit From 05d557a5cf59b08590cfc1bfc44bfdc0f9ac9681 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:33 +0200 Subject: arm64: kvm: add prototypes for functions called in asm A lot of kvm specific functions are called only from assembler and have no extern prototype, but that causes a W=1 warnings: arch/arm64/kvm/handle_exit.c:365:24: error: no previous prototype for 'nvhe_hyp_panic_handler' [-Werror=missing-prototypes] arch/arm64/kvm/va_layout.c:188:6: error: no previous prototype for 'kvm_patch_vector_branch' [-Werror=mi ssing-prototypes] arch/arm64/kvm/va_layout.c:287:6: error: no previous prototype for 'kvm_get_kimage_voffset' [-Werror=mis sing-prototypes] arch/arm64/kvm/va_layout.c:293:6: error: no previous prototype for 'kvm_compute_final_ctr_el0' [-Werror= missing-prototypes] arch/arm64/kvm/hyp/vhe/switch.c:259:17: error: no previous prototype for 'hyp_panic' [-Werror=missing-pr arch/arm64/kvm/hyp/nvhe/switch.c:389:17: error: no previous prototype for 'kvm_unexpected_el2_exception' arch/arm64/kvm/hyp/nvhe/switch.c:384:28: error: no previous prototype for 'hyp_panic_bad_stack' [-Werror arch/arm64/kvm/hyp/nvhe/hyp-main.c:383:6: error: no previous prototype for 'handle_trap' [-Werror=missin arch/arm64/kvm/hyp/nvhe/psci-relay.c:203:28: error: no previous prototype for 'kvm_host_psci_cpu_entry' [-Werror=missing-prototypes] Declare those in asm/kvm_asm.h, which already has related declarations. Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-7-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/kvm_asm.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 43c3bc0f9544..86042afa86c3 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -267,6 +267,24 @@ extern u64 __kvm_get_mdcr_el2(void); __kvm_at_err; \ } ) +void __noreturn hyp_panic(void); +asmlinkage void kvm_unexpected_el2_exception(void); +asmlinkage void __noreturn hyp_panic(void); +asmlinkage void __noreturn hyp_panic_bad_stack(void); +asmlinkage void kvm_unexpected_el2_exception(void); +struct kvm_cpu_context; +void handle_trap(struct kvm_cpu_context *host_ctxt); +asmlinkage void __noreturn kvm_host_psci_cpu_entry(bool is_cpu_on); +void __noreturn __pkvm_init_finalise(void); +void kvm_nvhe_prepare_backtrace(unsigned long fp, unsigned long pc); +void kvm_patch_vector_branch(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst); +void kvm_get_kimage_voffset(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst); +void kvm_compute_final_ctr_el0(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst); +void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr, u64 elr_virt, + u64 elr_phys, u64 par, uintptr_t vcpu, u64 far, u64 hpfar); #else /* __ASSEMBLY__ */ -- cgit From 68a879b55346588de936c99a04e665d9cdc326e6 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:34 +0200 Subject: arm64: cpuidle: fix #ifdef for acpi functions The acpi_processor_ffh_lpi_* functions are defined whenever CONFIG_ACPI is enabled, but only called and declared when CONFIG_ACPI_PROCESSOR_IDLE is also enabled. Without that, a W=1 build triggers missing-prototope warnings, so the #ifdef needs to be adapted: arch/arm64/kernel/cpuidle.c:60:5: error: no previous prototype for 'acpi_processor_ffh_lpi_probe' [-Werror=missing-prototypes] arch/arm64/kernel/cpuidle.c:65:15: error: no previous prototype for 'acpi_processor_ffh_lpi_enter' [-Werror=missing-prototypes] Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-8-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/kernel/cpuidle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kernel/cpuidle.c b/arch/arm64/kernel/cpuidle.c index 42e19fff40ee..d1f68599c29f 100644 --- a/arch/arm64/kernel/cpuidle.c +++ b/arch/arm64/kernel/cpuidle.c @@ -13,7 +13,7 @@ #include #include -#ifdef CONFIG_ACPI +#ifdef CONFIG_ACPI_PROCESSOR_IDLE #include -- cgit From fbc0cd6f60443264af0b7aed050cae2ef38036f9 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:35 +0200 Subject: arm64: efi: add efi_handle_corrupted_x18 prototype This functions is only called from assembler and lacks a prototype, which is seen from this W=1 warning: arch/arm64/kernel/efi.c:155:25: error: no previous prototype for 'efi_handle_corrupted_x18' [-Werror=missing-prototypes] Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-9-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/efi.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index f86b157a5da3..ef46f2daca62 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -166,4 +166,6 @@ static inline void efi_capsule_flush_cache_range(void *addr, int size) dcache_clean_inval_poc((unsigned long)addr, (unsigned long)addr + size); } +efi_status_t efi_handle_corrupted_x18(efi_status_t s, const char *f); + #endif /* _ASM_EFI_H */ -- cgit From b925b4314c9155b32d17e9dfda37d64c229063b7 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:36 +0200 Subject: arm64: hide unused is_valid_bugaddr() When generic BUG() support is disabled, this function has no declaration and no callers but causes a W=1 warning: arch/arm64/kernel/traps.c:950:5: error: no previous prototype for 'is_valid_bugaddr' [-Werror=missing-prototypes] Add an #ifdef that matches the one around the declaration. Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-10-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/kernel/traps.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 4bb1b8f47298..720d3780d8fd 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -947,7 +947,7 @@ void do_serror(struct pt_regs *regs, unsigned long esr) } /* GENERIC_BUG traps */ - +#ifdef CONFIG_GENERIC_BUG int is_valid_bugaddr(unsigned long addr) { /* @@ -959,6 +959,7 @@ int is_valid_bugaddr(unsigned long addr) */ return 1; } +#endif static int bug_handler(struct pt_regs *regs, unsigned long esr) { -- cgit From 60a0aab7463ee69296692d980b96510ccce3934e Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:37 +0200 Subject: arm64: module-plts: inline linux/moduleloader.h module_frob_arch_sections() is declared in moduleloader.h, but that is not included before the definition: arch/arm64/kernel/module-plts.c:286:5: error: no previous prototype for 'module_frob_arch_sections' [-Werror=missing-prototypes] Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-11-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/kernel/module-plts.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kernel/module-plts.c b/arch/arm64/kernel/module-plts.c index 543493bf924d..ad02058756b5 100644 --- a/arch/arm64/kernel/module-plts.c +++ b/arch/arm64/kernel/module-plts.c @@ -7,6 +7,7 @@ #include #include #include +#include #include static struct plt_entry __get_adrp_add_pair(u64 dst, u64 pc, -- cgit From 1a1183938946fc1e06425d830a85e5aad63c049d Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:38 +0200 Subject: arm64: flush: include linux/libnvdimm.h The two cache management functions are declared in libnvdimm.h but provided by architecture specific code. Without including the header, this causes a W=1 warning: arch/arm64/mm/flush.c:96:6: error: no previous prototype for 'arch_wb_cache_pmem' [-Werror=missing-prototypes] arch/arm64/mm/flush.c:104:6: error: no previous prototype for 'arch_invalidate_pmem' [-Werror=missing-prototypes] Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-12-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/mm/flush.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c index 5f9379b3c8c8..4e6476094952 100644 --- a/arch/arm64/mm/flush.c +++ b/arch/arm64/mm/flush.c @@ -8,6 +8,7 @@ #include #include +#include #include #include -- cgit From a7f5cb606e9993daf2d129efc5e3b6ca46ad9227 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:39 +0200 Subject: arm64: kaslr: add kaslr_early_init() declaration kaslr_early_init() is called from assembler code and does not need a declaration to work, but adding one anyway shuts up this W=1 warning: arch/arm64/kernel/pi/kaslr_early.c:88:16: error: no previous prototype for 'kaslr_early_init' [-Werror=missing-prototypes] Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-13-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/archrandom.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/archrandom.h b/arch/arm64/include/asm/archrandom.h index 2f5f3da34782..b0abc64f86b0 100644 --- a/arch/arm64/include/asm/archrandom.h +++ b/arch/arm64/include/asm/archrandom.h @@ -129,4 +129,6 @@ static inline bool __init __early_cpu_has_rndr(void) return (ftr >> ID_AA64ISAR0_EL1_RNDR_SHIFT) & 0xf; } +u64 kaslr_early_init(void *fdt); + #endif /* _ASM_ARCHRANDOM_H */ -- cgit From 8ada7aab02ee9b07c8539a5c9cc452520b183a72 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:40 +0200 Subject: arm64: signal: include asm/exception.h The do_notify_resume() is in a header that is not included for the definition, which causes a W=1 warning: arch/arm64/kernel/signal.c:1280:6: error: no previous prototype for 'do_notify_resume' [-Werror=missing-prototypes] Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-14-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/kernel/signal.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 2cfc810d0a5b..3457906ba117 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include -- cgit From e13d32e99264e0b63b01417e2f2db627f4507b97 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:41 +0200 Subject: arm64: move early_brk64 prototype to header The prototype used for calling early_brk64() is in the file that calls it, which is the wrong place, as it is not included for the definition: arch/arm64/kernel/traps.c:1100:12: error: no previous prototype for 'early_brk64' [-Werror=missing-prototypes] Move it to an appropriate header instead. Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-15-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/traps.h | 2 ++ arch/arm64/mm/fault.c | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/traps.h b/arch/arm64/include/asm/traps.h index 1f361e2da516..d66dfb3a72dd 100644 --- a/arch/arm64/include/asm/traps.h +++ b/arch/arm64/include/asm/traps.h @@ -29,6 +29,8 @@ void arm64_force_sig_fault(int signo, int code, unsigned long far, const char *s void arm64_force_sig_mceerr(int code, unsigned long far, short lsb, const char *str); void arm64_force_sig_ptrace_errno_trap(int errno, unsigned long far, const char *str); +int early_brk64(unsigned long addr, unsigned long esr, struct pt_regs *regs); + /* * Move regs->pc to next instruction and do necessary setup before it * is executed. diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index cb21ccd7940d..e99eacebb6c1 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -886,9 +886,6 @@ void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs) } NOKPROBE_SYMBOL(do_sp_pc_abort); -int __init early_brk64(unsigned long addr, unsigned long esr, - struct pt_regs *regs); - /* * __refdata because early_brk64 is __init, but the reference to it is * clobbered at arch_initcall time. -- cgit From c152aed4dcc21e6d496e700148fdd85c2b0ff09c Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 May 2023 18:06:42 +0200 Subject: arm64: add alt_cb_patch_nops prototype alt_cb_patch_nops() is called through an inline asm macro, so it does not need a prototype for the caller, but adding it avoids this W=1 build warning: arch/arm64/kernel/alternative.c:295:14: error: no previous prototype for 'alt_cb_patch_nops' [-Werror=missing-prototypes] Signed-off-by: Arnd Bergmann Reviewed-by: Kees Cook Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20230516160642.523862-16-arnd@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/alternative.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h index a38b92e11811..55da3fedcfe2 100644 --- a/arch/arm64/include/asm/alternative.h +++ b/arch/arm64/include/asm/alternative.h @@ -31,5 +31,8 @@ void apply_alternatives_module(void *start, size_t length); static inline void apply_alternatives_module(void *start, size_t length) { } #endif +void alt_cb_patch_nops(struct alt_instr *alt, __le32 *origptr, + __le32 *updptr, int nr_inst); + #endif /* __ASSEMBLY__ */ #endif /* __ASM_ALTERNATIVE_H */ -- cgit From de847275449a99343393a5f2a4179cf7f4d12372 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Thu, 11 May 2023 15:05:14 +0900 Subject: arm64/esr: Use GENMASK() for the ISS mask We express the mask for ESR_ELx.ISS in a non-standard manner, not using the standard helpers. In preparation for adding decode for ISS2 convert to use GENMASK(). No functional change. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20230417-arm64-iss2-dabt-decode-v3-1-c1fa503e503a@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/esr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 8487aec9b658..0bd879007168 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -75,7 +75,7 @@ #define ESR_ELx_IL_SHIFT (25) #define ESR_ELx_IL (UL(1) << ESR_ELx_IL_SHIFT) -#define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1) +#define ESR_ELx_ISS_MASK (GENMASK(24, 0)) #define ESR_ELx_ISS(esr) ((esr) & ESR_ELx_ISS_MASK) /* ISS field definitions shared by different classes */ -- cgit From 1f9d4ba6839cc77717ed603fc6df1f36995da76d Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Thu, 11 May 2023 15:05:15 +0900 Subject: arm64/esr: Add decode of ISS2 to data abort reporting The architecture has added more information about faults to ISS2 within ESR. Add decode of this to our data abort fault decode to aid diagnostics. Features that are not currently enabled are included here for completeness. Since the architecture specifies the values of bits within ISS2 in terms of ISS2 rather than in terms of the register as a whole we do so for our definitions as well, this makes it easier to review bitfield definitions. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20230417-arm64-iss2-dabt-decode-v3-2-c1fa503e503a@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/esr.h | 17 +++++++++++++++++ arch/arm64/mm/fault.c | 17 ++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 0bd879007168..0552a29f026b 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -77,6 +77,9 @@ #define ESR_ELx_IL (UL(1) << ESR_ELx_IL_SHIFT) #define ESR_ELx_ISS_MASK (GENMASK(24, 0)) #define ESR_ELx_ISS(esr) ((esr) & ESR_ELx_ISS_MASK) +#define ESR_ELx_ISS2_SHIFT (32) +#define ESR_ELx_ISS2_MASK (GENMASK_ULL(55, 32)) +#define ESR_ELx_ISS2(esr) (((esr) & ESR_ELx_ISS2_MASK) >> ESR_ELx_ISS2_SHIFT) /* ISS field definitions shared by different classes */ #define ESR_ELx_WNR_SHIFT (6) @@ -140,6 +143,20 @@ #define ESR_ELx_CM_SHIFT (8) #define ESR_ELx_CM (UL(1) << ESR_ELx_CM_SHIFT) +/* ISS2 field definitions for Data Aborts */ +#define ESR_ELx_TnD_SHIFT (10) +#define ESR_ELx_TnD (UL(1) << ESR_ELx_TnD_SHIFT) +#define ESR_ELx_TagAccess_SHIFT (9) +#define ESR_ELx_TagAccess (UL(1) << ESR_ELx_TagAccess_SHIFT) +#define ESR_ELx_GCS_SHIFT (8) +#define ESR_ELx_GCS (UL(1) << ESR_ELx_GCS_SHIFT) +#define ESR_ELx_Overlay_SHIFT (6) +#define ESR_ELx_Overlay (UL(1) << ESR_ELx_Overlay_SHIFT) +#define ESR_ELx_DirtyBit_SHIFT (5) +#define ESR_ELx_DirtyBit (UL(1) << ESR_ELx_DirtyBit_SHIFT) +#define ESR_ELx_Xs_SHIFT (0) +#define ESR_ELx_Xs_MASK (GENMASK_ULL(4, 0)) + /* ISS field definitions for exceptions taken in to Hyp */ #define ESR_ELx_CV (UL(1) << 24) #define ESR_ELx_COND_SHIFT (20) diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index cb21ccd7940d..fb5e0fb8b2a7 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -66,6 +66,8 @@ static inline const struct fault_info *esr_to_debug_fault_info(unsigned long esr static void data_abort_decode(unsigned long esr) { + unsigned long iss2 = ESR_ELx_ISS2(esr); + pr_alert("Data abort info:\n"); if (esr & ESR_ELx_ISV) { @@ -78,12 +80,21 @@ static void data_abort_decode(unsigned long esr) (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT, (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT); } else { - pr_alert(" ISV = 0, ISS = 0x%08lx\n", esr & ESR_ELx_ISS_MASK); + pr_alert(" ISV = 0, ISS = 0x%08lx, ISS2 = 0x%08lx\n", + esr & ESR_ELx_ISS_MASK, iss2); } - pr_alert(" CM = %lu, WnR = %lu\n", + pr_alert(" CM = %lu, WnR = %lu, TnD = %lu, TagAccess = %lu\n", (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT, - (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT); + (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT, + (iss2 & ESR_ELx_TnD) >> ESR_ELx_TnD_SHIFT, + (iss2 & ESR_ELx_TagAccess) >> ESR_ELx_TagAccess_SHIFT); + + pr_alert(" GCS = %ld, Overlay = %lu, DirtyBit = %lu, Xs = %llu\n", + (iss2 & ESR_ELx_GCS) >> ESR_ELx_GCS_SHIFT, + (iss2 & ESR_ELx_Overlay) >> ESR_ELx_Overlay_SHIFT, + (iss2 & ESR_ELx_DirtyBit) >> ESR_ELx_DirtyBit_SHIFT, + (iss2 & ESR_ELx_Xs_MASK) >> ESR_ELx_Xs_SHIFT); } static void mem_abort_decode(unsigned long esr) -- cgit From cb5aa637943857f7f937a51d1e621dbe925f9f67 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 22 May 2023 15:28:00 +0100 Subject: kselftest/arm64: Add a smoke test for ptracing hardware break/watch points There was a report that the hardware breakpoints and watch points weren't reporting the debug architecture version as expected, they were reporting a version of 0 which is not defined in the architecture. This happens when running in a KVM guest if the host has a debug architecture version not supported by KVM, it in turn confuses GDB which rejects any debug architecture version it does not know about. Add a test that covers that situation and while we're at it reports the debug architecture version and number of slots available to aid with figuring out problems that may arise. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20230414-arm64-test-hw-breakpoint-v2-1-90a19e3b1059@kernel.org Signed-off-by: Catalin Marinas --- tools/testing/selftests/arm64/abi/ptrace.c | 32 +++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/arm64/abi/ptrace.c b/tools/testing/selftests/arm64/abi/ptrace.c index be952511af22..abe4d58d731d 100644 --- a/tools/testing/selftests/arm64/abi/ptrace.c +++ b/tools/testing/selftests/arm64/abi/ptrace.c @@ -20,7 +20,7 @@ #include "../../kselftest.h" -#define EXPECTED_TESTS 7 +#define EXPECTED_TESTS 11 #define MAX_TPIDRS 2 @@ -132,6 +132,34 @@ static void test_tpidr(pid_t child) } } +static void test_hw_debug(pid_t child, int type, const char *type_name) +{ + struct user_hwdebug_state state; + struct iovec iov; + int slots, arch, ret; + + iov.iov_len = sizeof(state); + iov.iov_base = &state; + + /* Should be able to read the values */ + ret = ptrace(PTRACE_GETREGSET, child, type, &iov); + ksft_test_result(ret == 0, "read_%s\n", type_name); + + if (ret == 0) { + /* Low 8 bits is the number of slots, next 4 bits the arch */ + slots = state.dbg_info & 0xff; + arch = (state.dbg_info >> 8) & 0xf; + + ksft_print_msg("%s version %d with %d slots\n", type_name, + arch, slots); + + /* Zero is not currently architecturally valid */ + ksft_test_result(arch, "%s_arch_set\n", type_name); + } else { + ksft_test_result_skip("%s_arch_set\n"); + } +} + static int do_child(void) { if (ptrace(PTRACE_TRACEME, -1, NULL, NULL)) @@ -207,6 +235,8 @@ static int do_parent(pid_t child) ksft_print_msg("Parent is %d, child is %d\n", getpid(), child); test_tpidr(child); + test_hw_debug(child, NT_ARM_HW_WATCH, "NT_ARM_HW_WATCH"); + test_hw_debug(child, NT_ARM_HW_BREAK, "NT_ARM_HW_BREAK"); ret = EXIT_SUCCESS; -- cgit From e34f78b970ea51d17841f6168e50223efc690c76 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 23 May 2023 22:49:00 +0100 Subject: arm64/cpufeature: Use helper for ECV CNTPOFF cpufeature The newly added support for ECV CNTPOFF open codes the recently added helper ARM64_CPUID_FIELDS(), make use of the helper. No functional change. Signed-off-by: Mark Brown Reviewed-by: Anshuman Khandual Link: https://lore.kernel.org/r/20230523-arm64-ecv-helper-v1-1-506dfb5fb199@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/kernel/cpufeature.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 7d7128c65161..27326f35b646 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2235,11 +2235,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .capability = ARM64_HAS_ECV_CNTPOFF, .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_cpuid_feature, - .sys_reg = SYS_ID_AA64MMFR0_EL1, - .field_pos = ID_AA64MMFR0_EL1_ECV_SHIFT, - .field_width = 4, - .sign = FTR_UNSIGNED, - .min_field_value = ID_AA64MMFR0_EL1_ECV_CNTPOFF, + ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, ECV, CNTPOFF) }, #ifdef CONFIG_ARM64_PAN { -- cgit From af94aad4c9150cca6781ad134c950fb05dff43f9 Mon Sep 17 00:00:00 2001 From: Kristina Martsenko Date: Tue, 9 May 2023 15:22:25 +0100 Subject: KVM: arm64: initialize HCRX_EL2 ARMv8.7/9.2 adds a new hypervisor configuration register HCRX_EL2. Initialize the register to a safe value (all fields 0), to be robust against firmware that has not initialized it. This is also needed to ensure that the register is reinitialized after a kexec by a future kernel. In addition, move SMPME setup over to the new flags, as it would otherwise get overridden. It is safe to set the bit even if SME is not (uniformly) supported, as it will write to a RES0 bit (having no effect), and SME will be disabled by the cpufeature framework. (Similar to how e.g. the API bit is handled in HCR_HOST_NVHE_FLAGS.) Signed-off-by: Kristina Martsenko Acked-by: Marc Zyngier Acked-by: Oliver Upton Link: https://lore.kernel.org/r/20230509142235.3284028-2-kristina.martsenko@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/el2_setup.h | 18 ++++++++++-------- arch/arm64/include/asm/kvm_arm.h | 3 +++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 037724b19c5c..0201577863ca 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -22,6 +22,15 @@ isb .endm +.macro __init_el2_hcrx + mrs x0, id_aa64mmfr1_el1 + ubfx x0, x0, #ID_AA64MMFR1_EL1_HCX_SHIFT, #4 + cbz x0, .Lskip_hcrx_\@ + mov_q x0, HCRX_HOST_FLAGS + msr_s SYS_HCRX_EL2, x0 +.Lskip_hcrx_\@: +.endm + /* * Allow Non-secure EL1 and EL0 to access physical timer and counter. * This is not necessary for VHE, since the host kernel runs in EL2, @@ -184,6 +193,7 @@ */ .macro init_el2_state __init_el2_sctlr + __init_el2_hcrx __init_el2_timers __init_el2_debug __init_el2_lor @@ -284,14 +294,6 @@ cbz x1, .Lskip_sme_\@ msr_s SYS_SMPRIMAP_EL2, xzr // Make all priorities equal - - mrs x1, id_aa64mmfr1_el1 // HCRX_EL2 present? - ubfx x1, x1, #ID_AA64MMFR1_EL1_HCX_SHIFT, #4 - cbz x1, .Lskip_sme_\@ - - mrs_s x1, SYS_HCRX_EL2 - orr x1, x1, #HCRX_EL2_SMPME_MASK // Enable priority mapping - msr_s SYS_HCRX_EL2, x1 .Lskip_sme_\@: .endm diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index baef29fcbeee..fb7fe28b8eb8 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -9,6 +9,7 @@ #include #include +#include #include /* Hyp Configuration Register (HCR) bits */ @@ -92,6 +93,8 @@ #define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC) #define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H) +#define HCRX_HOST_FLAGS (HCRX_EL2_SMPME) + /* TCR_EL2 Registers bits */ #define TCR_EL2_RES1 ((1U << 31) | (1 << 23)) #define TCR_EL2_TBI (1 << 20) -- cgit From b0c756fe996ac930033882ca56410639e5cad1ec Mon Sep 17 00:00:00 2001 From: Kristina Martsenko Date: Tue, 9 May 2023 15:22:26 +0100 Subject: arm64: cpufeature: detect FEAT_HCX Detect if the system has the new HCRX_EL2 register added in ARMv8.7/9.2, so that subsequent patches can check for its presence. KVM currently relies on the register being present on all CPUs (or none), so the kernel will panic if that is not the case. Fortunately no such systems currently exist, but this can be revisited if they appear. Note that the kernel will not panic if CONFIG_KVM is disabled. Reviewed-by: Catalin Marinas Signed-off-by: Kristina Martsenko Link: https://lore.kernel.org/r/20230509142235.3284028-3-kristina.martsenko@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/kernel/cpufeature.c | 8 ++++++++ arch/arm64/tools/cpucaps | 1 + 2 files changed, 9 insertions(+) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 7d7128c65161..9898ad77b1db 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -364,6 +364,7 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = { static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = { ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TIDCP1_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HCX_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_ETS_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TWED_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_XNX_SHIFT, 4, 0), @@ -2309,6 +2310,13 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = is_kvm_protected_mode, }, + { + .desc = "HCRX_EL2 register", + .capability = ARM64_HAS_HCX, + .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, + .matches = has_cpuid_feature, + ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HCX, IMP) + }, #endif { .desc = "Kernel page table isolation (KPTI)", diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 40ba95472594..e1de10fa080e 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -32,6 +32,7 @@ HAS_GENERIC_AUTH_IMP_DEF HAS_GIC_CPUIF_SYSREGS HAS_GIC_PRIO_MASKING HAS_GIC_PRIO_RELAXED_SYNC +HAS_HCX HAS_LDAPR HAS_LSE_ATOMICS HAS_NESTED_VIRT -- cgit From 306b4c9f7120c485607cbbfa1ac3ecec005d8231 Mon Sep 17 00:00:00 2001 From: Kristina Martsenko Date: Tue, 9 May 2023 15:22:27 +0100 Subject: KVM: arm64: switch HCRX_EL2 between host and guest Switch the HCRX_EL2 register between host and guest configurations, in order to enable different features in the host and guest. Now that there are separate guest flags, we can also remove SMPME from the host flags, as SMPME is used for virtualizing SME priorities and has no use in the host. Signed-off-by: Kristina Martsenko Acked-by: Marc Zyngier Acked-by: Oliver Upton Link: https://lore.kernel.org/r/20230509142235.3284028-4-kristina.martsenko@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/kvm_arm.h | 3 ++- arch/arm64/kvm/hyp/include/hyp/switch.h | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index fb7fe28b8eb8..7bb2fbddda54 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -93,7 +93,8 @@ #define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC) #define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H) -#define HCRX_HOST_FLAGS (HCRX_EL2_SMPME) +#define HCRX_GUEST_FLAGS (HCRX_EL2_SMPME) +#define HCRX_HOST_FLAGS 0 /* TCR_EL2 Registers bits */ #define TCR_EL2_RES1 ((1U << 31) | (1 << 23)) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index e78a08a72a3c..eb123aa7479d 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -130,6 +130,9 @@ static inline void ___activate_traps(struct kvm_vcpu *vcpu) if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE)) write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2); + + if (cpus_have_final_cap(ARM64_HAS_HCX)) + write_sysreg_s(HCRX_GUEST_FLAGS, SYS_HCRX_EL2); } static inline void ___deactivate_traps(struct kvm_vcpu *vcpu) @@ -144,6 +147,9 @@ static inline void ___deactivate_traps(struct kvm_vcpu *vcpu) vcpu->arch.hcr_el2 &= ~HCR_VSE; vcpu->arch.hcr_el2 |= read_sysreg(hcr_el2) & HCR_VSE; } + + if (cpus_have_final_cap(ARM64_HAS_HCX)) + write_sysreg_s(HCRX_HOST_FLAGS, SYS_HCRX_EL2); } static inline bool __populate_fault_info(struct kvm_vcpu *vcpu) -- cgit From f32c053b9806e42209d40e9ce7ed6f7f8be3be3b Mon Sep 17 00:00:00 2001 From: Kristina Martsenko Date: Tue, 9 May 2023 15:22:28 +0100 Subject: arm64: mops: document boot requirements for MOPS FEAT_MOPS introduces new instructions, we require that these instructions not execute as UNDEFINED when we identify that the feature is supported. Signed-off-by: Kristina Martsenko Link: https://lore.kernel.org/r/20230509142235.3284028-5-kristina.martsenko@arm.com Signed-off-by: Catalin Marinas --- Documentation/arm64/booting.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/arm64/booting.rst b/Documentation/arm64/booting.rst index ffeccdd6bdac..b3bbf330ed0a 100644 --- a/Documentation/arm64/booting.rst +++ b/Documentation/arm64/booting.rst @@ -379,6 +379,12 @@ Before jumping into the kernel, the following conditions must be met: - SMCR_EL2.EZT0 (bit 30) must be initialised to 0b1. + For CPUs with Memory Copy and Memory Set instructions (FEAT_MOPS): + + - If the kernel is entered at EL1 and EL2 is present: + + - HCRX_EL2.MSCEn (bit 11) must be initialised to 0b1. + The requirements described above for CPU mode, caches, MMUs, architected timers, coherency and system registers apply to all CPUs. All CPUs must enter the kernel in the same exception level. Where the values documented -- cgit From b1319c0e955933eae918a8a95c5361378ca86968 Mon Sep 17 00:00:00 2001 From: Kristina Martsenko Date: Tue, 9 May 2023 15:22:29 +0100 Subject: arm64: mops: don't disable host MOPS instructions from EL2 To allow nVHE host EL0 and EL1 to use FEAT_MOPS instructions, configure EL2 to not cause these instructions to be treated as UNDEFINED. A VHE host is unaffected by this control. Reviewed-by: Catalin Marinas Signed-off-by: Kristina Martsenko Link: https://lore.kernel.org/r/20230509142235.3284028-6-kristina.martsenko@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/kvm_arm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 7bb2fbddda54..d2d4f4cd12b8 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -94,7 +94,7 @@ #define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H) #define HCRX_GUEST_FLAGS (HCRX_EL2_SMPME) -#define HCRX_HOST_FLAGS 0 +#define HCRX_HOST_FLAGS (HCRX_EL2_MSCEn) /* TCR_EL2 Registers bits */ #define TCR_EL2_RES1 ((1U << 31) | (1 << 23)) -- cgit From 3172613fbcbb0634d91d05601f029da0c1466999 Mon Sep 17 00:00:00 2001 From: Kristina Martsenko Date: Tue, 9 May 2023 15:22:30 +0100 Subject: KVM: arm64: hide MOPS from guests As FEAT_MOPS is not supported in guests yet, hide it from the ID registers for guests. The MOPS instructions are UNDEFINED in guests as HCRX_EL2.MSCEn is not set in HCRX_GUEST_FLAGS, and will take an exception to EL1 if executed. Signed-off-by: Kristina Martsenko Acked-by: Catalin Marinas Acked-by: Marc Zyngier Acked-by: Oliver Upton Link: https://lore.kernel.org/r/20230509142235.3284028-7-kristina.martsenko@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/kvm/sys_regs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 71b12094d613..6dae7fe91cfa 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1252,6 +1252,7 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3)); if (!cpus_have_final_cap(ARM64_HAS_WFXT)) val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_WFxT); + val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_MOPS); break; case SYS_ID_AA64DFR0_EL1: /* Limit debug to ARMv8.0 */ -- cgit From 8536ceaa747174ded7983f13906b225e0c33ac51 Mon Sep 17 00:00:00 2001 From: Kristina Martsenko Date: Tue, 9 May 2023 15:22:31 +0100 Subject: arm64: mops: handle MOPS exceptions The memory copy/set instructions added as part of FEAT_MOPS can take an exception (e.g. page fault) part-way through their execution and resume execution afterwards. If however the task is re-scheduled and execution resumes on a different CPU, then the CPU may take a new type of exception to indicate this. This is because the architecture allows two options (Option A and Option B) to implement the instructions and a heterogeneous system can have different implementations between CPUs. In this case the OS has to reset the registers and restart execution from the prologue instruction. The algorithm for doing this is provided as part of the Arm ARM. Add an exception handler for the new exception and wire it up for userspace tasks. Reviewed-by: Catalin Marinas Signed-off-by: Kristina Martsenko Link: https://lore.kernel.org/r/20230509142235.3284028-8-kristina.martsenko@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/esr.h | 11 +++++++- arch/arm64/include/asm/exception.h | 1 + arch/arm64/kernel/entry-common.c | 11 ++++++++ arch/arm64/kernel/traps.c | 52 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 8487aec9b658..ca954f566861 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -47,7 +47,7 @@ #define ESR_ELx_EC_DABT_LOW (0x24) #define ESR_ELx_EC_DABT_CUR (0x25) #define ESR_ELx_EC_SP_ALIGN (0x26) -/* Unallocated EC: 0x27 */ +#define ESR_ELx_EC_MOPS (0x27) #define ESR_ELx_EC_FP_EXC32 (0x28) /* Unallocated EC: 0x29 - 0x2B */ #define ESR_ELx_EC_FP_EXC64 (0x2C) @@ -356,6 +356,15 @@ #define ESR_ELx_SME_ISS_ZA_DISABLED 3 #define ESR_ELx_SME_ISS_ZT_DISABLED 4 +/* ISS field definitions for MOPS exceptions */ +#define ESR_ELx_MOPS_ISS_MEM_INST (UL(1) << 24) +#define ESR_ELx_MOPS_ISS_FROM_EPILOGUE (UL(1) << 18) +#define ESR_ELx_MOPS_ISS_WRONG_OPTION (UL(1) << 17) +#define ESR_ELx_MOPS_ISS_OPTION_A (UL(1) << 16) +#define ESR_ELx_MOPS_ISS_DESTREG(esr) (((esr) & (UL(0x1f) << 10)) >> 10) +#define ESR_ELx_MOPS_ISS_SRCREG(esr) (((esr) & (UL(0x1f) << 5)) >> 5) +#define ESR_ELx_MOPS_ISS_SIZEREG(esr) (((esr) & (UL(0x1f) << 0)) >> 0) + #ifndef __ASSEMBLY__ #include diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index e73af709cb7a..72e83af0135f 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -77,6 +77,7 @@ void do_el0_svc(struct pt_regs *regs); void do_el0_svc_compat(struct pt_regs *regs); void do_el0_fpac(struct pt_regs *regs, unsigned long esr); void do_el1_fpac(struct pt_regs *regs, unsigned long esr); +void do_el0_mops(struct pt_regs *regs, unsigned long esr); void do_serror(struct pt_regs *regs, unsigned long esr); void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags); diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 3af3c01c93a6..a8ec174e5b0e 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -611,6 +611,14 @@ static void noinstr el0_bti(struct pt_regs *regs) exit_to_user_mode(regs); } +static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr) +{ + enter_from_user_mode(regs); + local_daif_restore(DAIF_PROCCTX); + do_el0_mops(regs, esr); + exit_to_user_mode(regs); +} + static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr) { enter_from_user_mode(regs); @@ -688,6 +696,9 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) case ESR_ELx_EC_BTI: el0_bti(regs); break; + case ESR_ELx_EC_MOPS: + el0_mops(regs, esr); + break; case ESR_ELx_EC_BREAKPT_LOW: case ESR_ELx_EC_SOFTSTP_LOW: case ESR_ELx_EC_WATCHPT_LOW: diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 4bb1b8f47298..32dc692bffd3 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -514,6 +514,57 @@ void do_el1_fpac(struct pt_regs *regs, unsigned long esr) die("Oops - FPAC", regs, esr); } +void do_el0_mops(struct pt_regs *regs, unsigned long esr) +{ + bool wrong_option = esr & ESR_ELx_MOPS_ISS_WRONG_OPTION; + bool option_a = esr & ESR_ELx_MOPS_ISS_OPTION_A; + int dstreg = ESR_ELx_MOPS_ISS_DESTREG(esr); + int srcreg = ESR_ELx_MOPS_ISS_SRCREG(esr); + int sizereg = ESR_ELx_MOPS_ISS_SIZEREG(esr); + unsigned long dst, src, size; + + dst = pt_regs_read_reg(regs, dstreg); + src = pt_regs_read_reg(regs, srcreg); + size = pt_regs_read_reg(regs, sizereg); + + /* + * Put the registers back in the original format suitable for a + * prologue instruction, using the generic return routine from the + * Arm ARM (DDI 0487I.a) rules CNTMJ and MWFQH. + */ + if (esr & ESR_ELx_MOPS_ISS_MEM_INST) { + /* SET* instruction */ + if (option_a ^ wrong_option) { + /* Format is from Option A; forward set */ + pt_regs_write_reg(regs, dstreg, dst + size); + pt_regs_write_reg(regs, sizereg, -size); + } + } else { + /* CPY* instruction */ + if (!(option_a ^ wrong_option)) { + /* Format is from Option B */ + if (regs->pstate & PSR_N_BIT) { + /* Backward copy */ + pt_regs_write_reg(regs, dstreg, dst - size); + pt_regs_write_reg(regs, srcreg, src - size); + } + } else { + /* Format is from Option A */ + if (size & BIT(63)) { + /* Forward copy */ + pt_regs_write_reg(regs, dstreg, dst + size); + pt_regs_write_reg(regs, srcreg, src + size); + pt_regs_write_reg(regs, sizereg, -size); + } + } + } + + if (esr & ESR_ELx_MOPS_ISS_FROM_EPILOGUE) + regs->pc -= 8; + else + regs->pc -= 4; +} + #define __user_cache_maint(insn, address, res) \ if (address >= TASK_SIZE_MAX) { \ res = -EFAULT; \ @@ -824,6 +875,7 @@ static const char *esr_class_str[] = { [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)", [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)", [ESR_ELx_EC_SP_ALIGN] = "SP Alignment", + [ESR_ELx_EC_MOPS] = "MOPS", [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)", [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)", [ESR_ELx_EC_SERROR] = "SError", -- cgit From 8cd076a67dc8eac5d613b3258f656efa7a54412e Mon Sep 17 00:00:00 2001 From: Kristina Martsenko Date: Tue, 9 May 2023 15:22:32 +0100 Subject: arm64: mops: handle single stepping after MOPS exception When a MOPS main or epilogue instruction is being executed, the task may get scheduled on a different CPU and restart execution from the prologue instruction. If the main or epilogue instruction is being single stepped then it makes sense to finish the step and take the step exception before starting to execute the next (prologue) instruction. So fast-forward the single step state machine when taking a MOPS exception. This means that if a main or epilogue instruction is single stepped with ptrace, the debugger will sometimes observe the PC moving back to the prologue instruction. (As already mentioned, this should be rare as it only happens when the task is scheduled to another CPU during the step.) This also ensures that perf breakpoints count prologue instructions consistently (i.e. every time they are executed), rather than skipping them when there also happens to be a breakpoint on a main or epilogue instruction. Acked-by: Catalin Marinas Signed-off-by: Kristina Martsenko Link: https://lore.kernel.org/r/20230509142235.3284028-9-kristina.martsenko@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/kernel/traps.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 32dc692bffd3..4363e3b53a81 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -563,6 +563,12 @@ void do_el0_mops(struct pt_regs *regs, unsigned long esr) regs->pc -= 8; else regs->pc -= 4; + + /* + * If single stepping then finish the step before executing the + * prologue instruction. + */ + user_fastforward_single_step(current); } #define __user_cache_maint(insn, address, res) \ -- cgit From b7564127ffcb1a26854d7515a0918d775e663639 Mon Sep 17 00:00:00 2001 From: Kristina Martsenko Date: Tue, 9 May 2023 15:22:33 +0100 Subject: arm64: mops: detect and enable FEAT_MOPS The Arm v8.8/9.3 FEAT_MOPS feature provides new instructions that perform a memory copy or set. Wire up the cpufeature code to detect the presence of FEAT_MOPS and enable it. Reviewed-by: Catalin Marinas Signed-off-by: Kristina Martsenko Link: https://lore.kernel.org/r/20230509142235.3284028-10-kristina.martsenko@arm.com Signed-off-by: Catalin Marinas --- Documentation/arm64/cpu-feature-registers.rst | 2 ++ Documentation/arm64/elf_hwcaps.rst | 3 +++ arch/arm64/include/asm/hwcap.h | 1 + arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/kernel/cpufeature.c | 15 +++++++++++++++ arch/arm64/kernel/cpuinfo.c | 1 + arch/arm64/tools/cpucaps | 1 + 7 files changed, 24 insertions(+) diff --git a/Documentation/arm64/cpu-feature-registers.rst b/Documentation/arm64/cpu-feature-registers.rst index c7adc7897df6..4e4625f2455f 100644 --- a/Documentation/arm64/cpu-feature-registers.rst +++ b/Documentation/arm64/cpu-feature-registers.rst @@ -288,6 +288,8 @@ infrastructure: +------------------------------+---------+---------+ | Name | bits | visible | +------------------------------+---------+---------+ + | MOPS | [19-16] | y | + +------------------------------+---------+---------+ | RPRES | [7-4] | y | +------------------------------+---------+---------+ | WFXT | [3-0] | y | diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst index 83e57e4d38e2..8f847d0dcf57 100644 --- a/Documentation/arm64/elf_hwcaps.rst +++ b/Documentation/arm64/elf_hwcaps.rst @@ -302,6 +302,9 @@ HWCAP2_SMEB16B16 HWCAP2_SMEF16F16 Functionality implied by ID_AA64SMFR0_EL1.F16F16 == 0b1 +HWCAP2_MOPS + Functionality implied by ID_AA64ISAR2_EL1.MOPS == 0b0001. + 4. Unused AT_HWCAP bits ----------------------- diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index 5d45f19fda7f..692b1ec663b2 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -137,6 +137,7 @@ #define KERNEL_HWCAP_SME_BI32I32 __khwcap2_feature(SME_BI32I32) #define KERNEL_HWCAP_SME_B16B16 __khwcap2_feature(SME_B16B16) #define KERNEL_HWCAP_SME_F16F16 __khwcap2_feature(SME_F16F16) +#define KERNEL_HWCAP_MOPS __khwcap2_feature(MOPS) /* * This yields a mask that user programs can use to figure out what diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 69a4fb749c65..a2cac4305b1e 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -102,5 +102,6 @@ #define HWCAP2_SME_BI32I32 (1UL << 40) #define HWCAP2_SME_B16B16 (1UL << 41) #define HWCAP2_SME_F16F16 (1UL << 42) +#define HWCAP2_MOPS (1UL << 43) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 9898ad77b1db..3badc4fa7154 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -223,6 +223,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_EL1_BC_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_MOPS_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH), FTR_STRICT, FTR_EXACT, ID_AA64ISAR2_EL1_APA3_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH), @@ -2187,6 +2188,11 @@ static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused) set_pstate_dit(1); } +static void cpu_enable_mops(const struct arm64_cpu_capabilities *__unused) +{ + sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_MSCEn); +} + /* Internal helper functions to match cpu capability type */ static bool cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap) @@ -2649,6 +2655,14 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_enable_dit, ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, DIT, IMP) }, + { + .desc = "Memory Copy and Memory Set instructions", + .capability = ARM64_HAS_MOPS, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_cpuid_feature, + .cpu_enable = cpu_enable_mops, + ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, MOPS, IMP) + }, {}, }; @@ -2777,6 +2791,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(ID_AA64ISAR2_EL1, RPRFM, IMP, CAP_HWCAP, KERNEL_HWCAP_RPRFM), HWCAP_CAP(ID_AA64ISAR2_EL1, RPRES, IMP, CAP_HWCAP, KERNEL_HWCAP_RPRES), HWCAP_CAP(ID_AA64ISAR2_EL1, WFxT, IMP, CAP_HWCAP, KERNEL_HWCAP_WFXT), + HWCAP_CAP(ID_AA64ISAR2_EL1, MOPS, IMP, CAP_HWCAP, KERNEL_HWCAP_MOPS), #ifdef CONFIG_ARM64_SME HWCAP_CAP(ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME), HWCAP_CAP(ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64), diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index eb4378c23b3c..076a124255d0 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -125,6 +125,7 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_SME_BI32I32] = "smebi32i32", [KERNEL_HWCAP_SME_B16B16] = "smeb16b16", [KERNEL_HWCAP_SME_F16F16] = "smef16f16", + [KERNEL_HWCAP_MOPS] = "mops", }; #ifdef CONFIG_COMPAT diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index e1de10fa080e..debc4609f129 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -35,6 +35,7 @@ HAS_GIC_PRIO_RELAXED_SYNC HAS_HCX HAS_LDAPR HAS_LSE_ATOMICS +HAS_MOPS HAS_NESTED_VIRT HAS_NO_FPSIMD HAS_NO_HW_PREFETCH -- cgit From 3e1dedb29d0f70b0bd10bb83001df8c04fc246eb Mon Sep 17 00:00:00 2001 From: Kristina Martsenko Date: Tue, 9 May 2023 15:22:34 +0100 Subject: arm64: mops: allow disabling MOPS from the kernel command line Make it possible to disable the MOPS extension at runtime using the kernel command line. This can be useful for testing or working around hardware issues. For example it could be used to test new memory copy routines that do not use MOPS instructions (e.g. from Arm Optimized Routines). Reviewed-by: Catalin Marinas Signed-off-by: Kristina Martsenko Link: https://lore.kernel.org/r/20230509142235.3284028-11-kristina.martsenko@arm.com Signed-off-by: Catalin Marinas --- Documentation/admin-guide/kernel-parameters.txt | 3 +++ arch/arm64/kernel/idreg-override.c | 2 ++ 2 files changed, 5 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 9e5bab29685f..e01fbfd78ae9 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -429,6 +429,9 @@ arm64.nosme [ARM64] Unconditionally disable Scalable Matrix Extension support + arm64.nomops [ARM64] Unconditionally disable Memory Copy and Memory + Set instructions support + ataflop= [HW,M68k] atarimouse= [HW,MOUSE] Atari Mouse diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c index 370ab84fd06e..8439248c21d3 100644 --- a/arch/arm64/kernel/idreg-override.c +++ b/arch/arm64/kernel/idreg-override.c @@ -123,6 +123,7 @@ static const struct ftr_set_desc isar2 __initconst = { .fields = { FIELD("gpa3", ID_AA64ISAR2_EL1_GPA3_SHIFT, NULL), FIELD("apa3", ID_AA64ISAR2_EL1_APA3_SHIFT, NULL), + FIELD("mops", ID_AA64ISAR2_EL1_MOPS_SHIFT, NULL), {} }, }; @@ -174,6 +175,7 @@ static const struct { "id_aa64isar1.gpi=0 id_aa64isar1.gpa=0 " "id_aa64isar1.api=0 id_aa64isar1.apa=0 " "id_aa64isar2.gpa3=0 id_aa64isar2.apa3=0" }, + { "arm64.nomops", "id_aa64isar2.mops=0" }, { "arm64.nomte", "id_aa64pfr1.mte=0" }, { "nokaslr", "kaslr.disabled=1" }, }; -- cgit From d8a324f102cc894fa0df6849504a9023f3ea5da6 Mon Sep 17 00:00:00 2001 From: Kristina Martsenko Date: Tue, 9 May 2023 15:22:35 +0100 Subject: kselftest/arm64: add MOPS to hwcap test Add the MOPS hwcap to the hwcap kselftest and check that a SIGILL is not generated when the feature is detected. A SIGILL is reliable when the feature is not detected as SCTLR_EL1.MSCEn won't have been set. Signed-off-by: Kristina Martsenko Reviewed-by: Mark Brown Link: https://lore.kernel.org/r/20230509142235.3284028-12-kristina.martsenko@arm.com Signed-off-by: Catalin Marinas --- tools/testing/selftests/arm64/abi/hwcap.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index 93333a90bf3a..d4ad813fed10 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -39,6 +39,20 @@ static void cssc_sigill(void) asm volatile(".inst 0xdac01c00" : : : "x0"); } +static void mops_sigill(void) +{ + char dst[1], src[1]; + register char *dstp asm ("x0") = dst; + register char *srcp asm ("x1") = src; + register long size asm ("x2") = 1; + + /* CPYP [x0]!, [x1]!, x2! */ + asm volatile(".inst 0x1d010440" + : "+r" (dstp), "+r" (srcp), "+r" (size) + : + : "cc", "memory"); +} + static void rng_sigill(void) { asm volatile("mrs x0, S3_3_C2_C4_0" : : : "x0"); @@ -209,6 +223,14 @@ static const struct hwcap_data { .cpuinfo = "cssc", .sigill_fn = cssc_sigill, }, + { + .name = "MOPS", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_MOPS, + .cpuinfo = "mops", + .sigill_fn = mops_sigill, + .sigill_reliable = true, + }, { .name = "RNG", .at_hwcap = AT_HWCAP2, -- cgit From 56b77ba112d48becc3e41c9fe2b1533ba220b7c3 Mon Sep 17 00:00:00 2001 From: Prathu Baronia Date: Sat, 27 May 2023 18:08:58 +0530 Subject: arm64/cpucaps: increase string width to properly format cpucaps.h The lengthiest capability is `WORKAROUND_TRBE_OVERWRITE_FILL_MODE` and its length is 35 characters so increase the width of left justified strings to 40 and adjust the tab space for `ARM64_NCAPS` accordingly. Now the generated cpucaps.h is properly formatted. Signed-off-by: Prathu Baronia Link: https://lore.kernel.org/r/20230527123900.680520-1-quic_pbaronia@quicinc.com Signed-off-by: Catalin Marinas --- arch/arm64/tools/gen-cpucaps.awk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/tools/gen-cpucaps.awk b/arch/arm64/tools/gen-cpucaps.awk index 00c9e72a200a..8525980379d7 100755 --- a/arch/arm64/tools/gen-cpucaps.awk +++ b/arch/arm64/tools/gen-cpucaps.awk @@ -24,12 +24,12 @@ BEGIN { } /^[vA-Z0-9_]+$/ { - printf("#define ARM64_%-30s\t%d\n", $0, cap_num++) + printf("#define ARM64_%-40s\t%d\n", $0, cap_num++) next } END { - printf("#define ARM64_NCAPS\t\t\t\t%d\n", cap_num) + printf("#define ARM64_NCAPS\t\t\t\t\t%d\n", cap_num) print "" print "#endif /* __ASM_CPUCAPS_H */" } -- cgit From ab1e29acdb33c971e1af8ed8ec427bd1deff5f32 Mon Sep 17 00:00:00 2001 From: Eric Chan Date: Wed, 31 May 2023 09:09:09 +0000 Subject: arm64: lockdep: enable checks for held locks when returning to userspace Currently arm64 doesn't use CONFIG_GENERIC_ENTRY and doesn't call lockdep_sys_exit() when returning to userspace. This means that lockdep won't check for held locks when returning to userspace, which would be useful to detect kernel bugs. Call lockdep_sys_exit() when returning to userspace, enabling checking for held locks. At the same time, rename arm64's prepare_exit_to_user_mode() to exit_to_user_mode_prepare() to more clearly align with the naming in the generic entry code. Signed-off-by: Eric Chan Acked-by: Mark Rutland Link: https://lore.kernel.org/r/20230531090909.357047-1-ericchancf@google.com Signed-off-by: Catalin Marinas --- arch/arm64/kernel/entry-common.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 3af3c01c93a6..9d6827201b6c 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -126,7 +126,7 @@ static __always_inline void __exit_to_user_mode(void) lockdep_hardirqs_on(CALLER_ADDR0); } -static __always_inline void prepare_exit_to_user_mode(struct pt_regs *regs) +static __always_inline void exit_to_user_mode_prepare(struct pt_regs *regs) { unsigned long flags; @@ -135,11 +135,13 @@ static __always_inline void prepare_exit_to_user_mode(struct pt_regs *regs) flags = read_thread_flags(); if (unlikely(flags & _TIF_WORK_MASK)) do_notify_resume(regs, flags); + + lockdep_sys_exit(); } static __always_inline void exit_to_user_mode(struct pt_regs *regs) { - prepare_exit_to_user_mode(regs); + exit_to_user_mode_prepare(regs); mte_check_tfsr_exit(); __exit_to_user_mode(); } -- cgit From 8339f7d8e178d9c933f437d14be0a5fd1359f53d Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 30 May 2023 12:03:23 +0100 Subject: arm64: module: remove old !KASAN_VMALLOC logic Historically, KASAN could be selected with or without KASAN_VMALLOC, and we had to be very careful where to place modules when KASAN_VMALLOC was not selected. However, since commit: f6f37d9320a11e90 ("arm64: select KASAN_VMALLOC for SW/HW_TAGS modes") Selecting CONFIG_KASAN on arm64 will also select CONFIG_KASAN_VMALLOC, and so the logic for handling CONFIG_KASAN without CONFIG_KASAN_VMALLOC is redundant and can be removed. Note: the "kasan.vmalloc={on,off}" option which only exists for HW_TAGS changes whether the vmalloc region is given non-match-all tags, and does not affect the page table manipulation code. The VM_DEFER_KMEMLEAK flag was only necessary for !CONFIG_KASAN_VMALLOC as described in its introduction in commit: 60115fa54ad7b913 ("mm: defer kmemleak object creation of module_alloc()") ... and therefore it can also be removed. Remove the redundant logic for !CONFIG_KASAN_VMALLOC. At the same time, add the missing braces around the multi-line conditional block in arch/arm64/kernel/module.c. Suggested-by: Ard Biesheuvel Signed-off-by: Mark Rutland Reviewed-by: Ard Biesheuvel Cc: Alexander Potapenko Cc: Andrew Morton Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Dmitry Vyukov Cc: Will Deacon Tested-by: Shanker Donthineni Link: https://lore.kernel.org/r/20230530110328.2213762-2-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/kernel/kaslr.c | 10 ---------- arch/arm64/kernel/module.c | 23 +++-------------------- 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c index e7477f21a4c9..df433c80c6ef 100644 --- a/arch/arm64/kernel/kaslr.c +++ b/arch/arm64/kernel/kaslr.c @@ -48,16 +48,6 @@ static int __init kaslr_init(void) pr_info("KASLR enabled\n"); - /* - * KASAN without KASAN_VMALLOC does not expect the module region to - * intersect the vmalloc region, since shadow memory is allocated for - * each module at load time, whereas the vmalloc region will already be - * shadowed by KASAN zero pages. - */ - BUILD_BUG_ON((IS_ENABLED(CONFIG_KASAN_GENERIC) || - IS_ENABLED(CONFIG_KASAN_SW_TAGS)) && - !IS_ENABLED(CONFIG_KASAN_VMALLOC)); - seed = get_random_u32(); if (IS_ENABLED(CONFIG_RANDOMIZE_MODULE_REGION_FULL)) { diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index 5af4975caeb5..b2657ac41226 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -32,33 +32,16 @@ void *module_alloc(unsigned long size) if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS)) gfp_mask |= __GFP_NOWARN; - if (IS_ENABLED(CONFIG_KASAN_GENERIC) || - IS_ENABLED(CONFIG_KASAN_SW_TAGS)) - /* don't exceed the static module region - see below */ - module_alloc_end = MODULES_END; - p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, - module_alloc_end, gfp_mask, PAGE_KERNEL, VM_DEFER_KMEMLEAK, + module_alloc_end, gfp_mask, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); - if (!p && IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && - (IS_ENABLED(CONFIG_KASAN_VMALLOC) || - (!IS_ENABLED(CONFIG_KASAN_GENERIC) && - !IS_ENABLED(CONFIG_KASAN_SW_TAGS)))) - /* - * KASAN without KASAN_VMALLOC can only deal with module - * allocations being served from the reserved module region, - * since the remainder of the vmalloc region is already - * backed by zero shadow pages, and punching holes into it - * is non-trivial. Since the module region is not randomized - * when KASAN is enabled without KASAN_VMALLOC, it is even - * less likely that the module region gets exhausted, so we - * can simply omit this fallback in that case. - */ + if (!p && IS_ENABLED(CONFIG_ARM64_MODULE_PLTS)) { p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, module_alloc_base + SZ_2G, GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); + } if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) { vfree(p); -- cgit From 55123afffe931ab02066f60084c9c495b9b52fda Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 30 May 2023 12:03:24 +0100 Subject: arm64: kasan: remove !KASAN_VMALLOC remnants Historically, KASAN could be selected with or without KASAN_VMALLOC, but since commit: f6f37d9320a11e90 ("arm64: select KASAN_VMALLOC for SW/HW_TAGS modes") ... we can never select KASAN without KASAN_VMALLOC on arm64, and thus arm64 code for KASAN && !KASAN_VMALLOC is redundant and can be removed. Remove the redundant code kasan_init.c Signed-off-by: Mark Rutland Reviewed-by: Alexander Potapenko Reviewed-by: Ard Biesheuvel Cc: Andrey Konovalov Cc: Andrey Ryabinin Cc: Dmitry Vyukov Cc: Will Deacon Tested-by: Shanker Donthineni Link: https://lore.kernel.org/r/20230530110328.2213762-3-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/mm/kasan_init.c | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c index e969e68de005..f17d066e85eb 100644 --- a/arch/arm64/mm/kasan_init.c +++ b/arch/arm64/mm/kasan_init.c @@ -214,7 +214,7 @@ static void __init clear_pgds(unsigned long start, static void __init kasan_init_shadow(void) { u64 kimg_shadow_start, kimg_shadow_end; - u64 mod_shadow_start, mod_shadow_end; + u64 mod_shadow_start; u64 vmalloc_shadow_end; phys_addr_t pa_start, pa_end; u64 i; @@ -223,7 +223,6 @@ static void __init kasan_init_shadow(void) kimg_shadow_end = PAGE_ALIGN((u64)kasan_mem_to_shadow(KERNEL_END)); mod_shadow_start = (u64)kasan_mem_to_shadow((void *)MODULES_VADDR); - mod_shadow_end = (u64)kasan_mem_to_shadow((void *)MODULES_END); vmalloc_shadow_end = (u64)kasan_mem_to_shadow((void *)VMALLOC_END); @@ -246,17 +245,9 @@ static void __init kasan_init_shadow(void) kasan_populate_early_shadow(kasan_mem_to_shadow((void *)PAGE_END), (void *)mod_shadow_start); - if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) { - BUILD_BUG_ON(VMALLOC_START != MODULES_END); - kasan_populate_early_shadow((void *)vmalloc_shadow_end, - (void *)KASAN_SHADOW_END); - } else { - kasan_populate_early_shadow((void *)kimg_shadow_end, - (void *)KASAN_SHADOW_END); - if (kimg_shadow_start > mod_shadow_end) - kasan_populate_early_shadow((void *)mod_shadow_end, - (void *)kimg_shadow_start); - } + BUILD_BUG_ON(VMALLOC_START != MODULES_END); + kasan_populate_early_shadow((void *)vmalloc_shadow_end, + (void *)KASAN_SHADOW_END); for_each_mem_range(i, &pa_start, &pa_end) { void *start = (void *)__phys_to_virt(pa_start); -- cgit From 6e13b6b923b35a965d128a40ef0c5d9dd101e603 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 30 May 2023 12:03:25 +0100 Subject: arm64: kaslr: split kaslr/module initialization Currently kaslr_init() handles a mixture of detecting/announcing whether KASLR is enabled, and randomizing the module region depending on whether KASLR is enabled. To make it easier to rework the module region initialization, split the KASLR initialization into two steps: * kaslr_init() determines whether KASLR should be enabled, and announces this choice, recording this to a new global boolean variable. This is called from setup_arch() just before the existing call to kaslr_requires_kpti() so that this will always provide the expected result. * kaslr_module_init() randomizes the module region when required. This is called as a subsys_initcall, where we previously called kaslr_init(). As a bonus, moving the KASLR reporting earlier makes it easier to spot and permits it to be logged via earlycon, making it easier to debug any issues that could be triggered by KASLR. Booting a v6.4-rc1 kernel with this patch applied, the log looks like: | EFI stub: Booting Linux Kernel... | EFI stub: Generating empty DTB | EFI stub: Exiting boot services... | [ 0.000000] Booting Linux on physical CPU 0x0000000000 [0x000f0510] | [ 0.000000] Linux version 6.4.0-rc1-00006-g4763a8f8aeb3 (mark@lakrids) (aarch64-linux-gcc (GCC) 12.1.0, GNU ld (GNU Binutils) 2.38) #2 SMP PREEMPT Tue May 9 11:03:37 BST 2023 | [ 0.000000] KASLR enabled | [ 0.000000] earlycon: pl11 at MMIO 0x0000000009000000 (options '') | [ 0.000000] printk: bootconsole [pl11] enabled Signed-off-by: Mark Rutland Reviewed-by: Ard Biesheuvel Cc: Will Deacon Tested-by: Shanker Donthineni Link: https://lore.kernel.org/r/20230530110328.2213762-4-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/memory.h | 14 ++++++++------ arch/arm64/kernel/kaslr.c | 39 +++++++++++++++++++++++++-------------- arch/arm64/kernel/setup.c | 2 ++ 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index c735afdf639b..215efc3bbbcf 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -204,15 +204,17 @@ static inline unsigned long kaslr_offset(void) return kimage_vaddr - KIMAGE_VADDR; } +#ifdef CONFIG_RANDOMIZE_BASE +void kaslr_init(void); static inline bool kaslr_enabled(void) { - /* - * The KASLR offset modulo MIN_KIMG_ALIGN is taken from the physical - * placement of the image rather than from the seed, so a displacement - * of less than MIN_KIMG_ALIGN means that no seed was provided. - */ - return kaslr_offset() >= MIN_KIMG_ALIGN; + extern bool __kaslr_is_enabled; + return __kaslr_is_enabled; } +#else +static inline void kaslr_init(void) { } +static inline bool kaslr_enabled(void) { return false; } +#endif /* * Allow all memory at the discovery stage. We will clip it later. diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c index df433c80c6ef..ce7079ba1dc1 100644 --- a/arch/arm64/kernel/kaslr.c +++ b/arch/arm64/kernel/kaslr.c @@ -25,28 +25,39 @@ u16 __initdata memstart_offset_seed; struct arm64_ftr_override kaslr_feature_override __initdata; -static int __init kaslr_init(void) -{ - u64 module_range; - u32 seed; - - /* - * Set a reasonable default for module_alloc_base in case - * we end up running with module randomization disabled. - */ - module_alloc_base = (u64)_etext - MODULES_VSIZE; +bool __ro_after_init __kaslr_is_enabled = false; +void __init kaslr_init(void) +{ if (kaslr_feature_override.val & kaslr_feature_override.mask & 0xf) { pr_info("KASLR disabled on command line\n"); - return 0; + return; } - if (!kaslr_enabled()) { + /* + * The KASLR offset modulo MIN_KIMG_ALIGN is taken from the physical + * placement of the image rather than from the seed, so a displacement + * of less than MIN_KIMG_ALIGN means that no seed was provided. + */ + if (kaslr_offset() < MIN_KIMG_ALIGN) { pr_warn("KASLR disabled due to lack of seed\n"); - return 0; + return; } pr_info("KASLR enabled\n"); + __kaslr_is_enabled = true; +} + +int kaslr_module_init(void) +{ + u64 module_range; + u32 seed; + + /* + * Set a reasonable default for module_alloc_base in case + * we end up running with module randomization disabled. + */ + module_alloc_base = (u64)_etext - MODULES_VSIZE; seed = get_random_u32(); @@ -80,4 +91,4 @@ static int __init kaslr_init(void) return 0; } -subsys_initcall(kaslr_init) +subsys_initcall(kaslr_module_init) diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index b8ec7b3ac9cb..417a8a86b2db 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -296,6 +296,8 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) *cmdline_p = boot_command_line; + kaslr_init(); + /* * If know now we are going to need KPTI then use non-global * mappings from the start, avoiding the cost of rewriting -- cgit From e46b7103aef39c3f421f0bff7a10ae5a29cd5cee Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 30 May 2023 12:03:26 +0100 Subject: arm64: module: move module randomization to module.c When CONFIG_RANDOMIZE_BASE=y, module_alloc_base is a variable which is configured by kaslr_module_init() in kaslr.c, and otherwise it is an expression defined in module.h. As kaslr_module_init() is no longer tightly coupled with the KASLR initialization code, we can centralize this in module.c. This patch moves kaslr_module_init() to module.c, making module_alloc_base a static variable, and removing redundant includes from kaslr.c. For the defintion of struct arm64_ftr_override we must include , which was previously included transitively via another header. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Reviewed-by: Ard Biesheuvel Cc: Will Deacon Tested-by: Shanker Donthineni Link: https://lore.kernel.org/r/20230530110328.2213762-5-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/module.h | 6 ----- arch/arm64/kernel/kaslr.c | 60 ++--------------------------------------- arch/arm64/kernel/module.c | 48 +++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 64 deletions(-) diff --git a/arch/arm64/include/asm/module.h b/arch/arm64/include/asm/module.h index 18734fed3bdd..3e7dcc5fa2f2 100644 --- a/arch/arm64/include/asm/module.h +++ b/arch/arm64/include/asm/module.h @@ -30,12 +30,6 @@ u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs, u64 module_emit_veneer_for_adrp(struct module *mod, Elf64_Shdr *sechdrs, void *loc, u64 val); -#ifdef CONFIG_RANDOMIZE_BASE -extern u64 module_alloc_base; -#else -#define module_alloc_base ((u64)_etext - MODULES_VSIZE) -#endif - struct plt_entry { /* * A program that conforms to the AArch64 Procedure Call Standard diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c index ce7079ba1dc1..17f96a19781d 100644 --- a/arch/arm64/kernel/kaslr.c +++ b/arch/arm64/kernel/kaslr.c @@ -4,23 +4,12 @@ */ #include -#include #include -#include -#include -#include -#include -#include -#include +#include -#include -#include +#include #include -#include -#include -#include -u64 __ro_after_init module_alloc_base; u16 __initdata memstart_offset_seed; struct arm64_ftr_override kaslr_feature_override __initdata; @@ -47,48 +36,3 @@ void __init kaslr_init(void) pr_info("KASLR enabled\n"); __kaslr_is_enabled = true; } - -int kaslr_module_init(void) -{ - u64 module_range; - u32 seed; - - /* - * Set a reasonable default for module_alloc_base in case - * we end up running with module randomization disabled. - */ - module_alloc_base = (u64)_etext - MODULES_VSIZE; - - seed = get_random_u32(); - - if (IS_ENABLED(CONFIG_RANDOMIZE_MODULE_REGION_FULL)) { - /* - * Randomize the module region over a 2 GB window covering the - * kernel. This reduces the risk of modules leaking information - * about the address of the kernel itself, but results in - * branches between modules and the core kernel that are - * resolved via PLTs. (Branches between modules will be - * resolved normally.) - */ - module_range = SZ_2G - (u64)(_end - _stext); - module_alloc_base = max((u64)_end - SZ_2G, (u64)MODULES_VADDR); - } else { - /* - * Randomize the module region by setting module_alloc_base to - * a PAGE_SIZE multiple in the range [_etext - MODULES_VSIZE, - * _stext) . This guarantees that the resulting region still - * covers [_stext, _etext], and that all relative branches can - * be resolved without veneers unless this region is exhausted - * and we fall back to a larger 2GB window in module_alloc() - * when ARM64_MODULE_PLTS is enabled. - */ - module_range = MODULES_VSIZE - (u64)(_etext - _stext); - } - - /* use the lower 21 bits to randomize the base of the module region */ - module_alloc_base += (module_range * (seed & ((1 << 21) - 1))) >> 21; - module_alloc_base &= PAGE_MASK; - - return 0; -} -subsys_initcall(kaslr_module_init) diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index b2657ac41226..51fd8abd9b74 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -15,13 +15,61 @@ #include #include #include +#include #include #include + #include #include #include #include +static u64 __ro_after_init module_alloc_base = (u64)_etext - MODULES_VSIZE; + +#ifdef CONFIG_RANDOMIZE_BASE +static int __init kaslr_module_init(void) +{ + u64 module_range; + u32 seed; + + if (!kaslr_enabled()) + return 0; + + seed = get_random_u32(); + + if (IS_ENABLED(CONFIG_RANDOMIZE_MODULE_REGION_FULL)) { + /* + * Randomize the module region over a 2 GB window covering the + * kernel. This reduces the risk of modules leaking information + * about the address of the kernel itself, but results in + * branches between modules and the core kernel that are + * resolved via PLTs. (Branches between modules will be + * resolved normally.) + */ + module_range = SZ_2G - (u64)(_end - _stext); + module_alloc_base = max((u64)_end - SZ_2G, (u64)MODULES_VADDR); + } else { + /* + * Randomize the module region by setting module_alloc_base to + * a PAGE_SIZE multiple in the range [_etext - MODULES_VSIZE, + * _stext) . This guarantees that the resulting region still + * covers [_stext, _etext], and that all relative branches can + * be resolved without veneers unless this region is exhausted + * and we fall back to a larger 2GB window in module_alloc() + * when ARM64_MODULE_PLTS is enabled. + */ + module_range = MODULES_VSIZE - (u64)(_etext - _stext); + } + + /* use the lower 21 bits to randomize the base of the module region */ + module_alloc_base += (module_range * (seed & ((1 << 21) - 1))) >> 21; + module_alloc_base &= PAGE_MASK; + + return 0; +} +subsys_initcall(kaslr_module_init) +#endif + void *module_alloc(unsigned long size) { u64 module_alloc_end = module_alloc_base + MODULES_VSIZE; -- cgit From ea3752ba9685b47db4571ddaee39344cf2b0bf45 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 30 May 2023 12:03:27 +0100 Subject: arm64: module: mandate MODULE_PLTS Contemporary kernels and modules can be relatively large, especially when common debug options are enabled. Using GCC 12.1.0, a v6.3-rc7 defconfig kernel is ~38M, and with PROVE_LOCKING + KASAN_INLINE enabled this expands to ~117M. Shanker reports [1] that the NVIDIA GPU driver alone can consume 110M of module space in some configurations. Both KASLR and ARM64_ERRATUM_843419 select MODULE_PLTS, so anyone wanting a kernel to have KASLR or run on Cortex-A53 will have MODULE_PLTS selected. This is the case in defconfig and distribution kernels (e.g. Debian, Android, etc). Practically speaking, this means we're very likely to need MODULE_PLTS and while it's almost guaranteed that MODULE_PLTS will be selected, it is possible to disable support, and we have to maintain some awkward special cases for such unusual configurations. This patch removes the MODULE_PLTS config option, with the support code always enabled if MODULES is selected. This results in a slight simplification, and will allow for further improvement in subsequent patches. For any config which currently selects MODULE_PLTS, there will be no functional change as a result of this patch. [1] https://lore.kernel.org/linux-arm-kernel/159ceeab-09af-3174-5058-445bc8dcf85b@nvidia.com/ Signed-off-by: Mark Rutland Reviewed-by: Ard Biesheuvel Cc: Shanker Donthineni Cc: Will Deacon Tested-by: Shanker Donthineni Link: https://lore.kernel.org/r/20230530110328.2213762-6-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/Kconfig | 28 +++------------------------- arch/arm64/include/asm/module.h | 2 -- arch/arm64/include/asm/module.lds.h | 2 -- arch/arm64/kernel/Makefile | 3 +-- arch/arm64/kernel/ftrace.c | 8 +++----- arch/arm64/kernel/module.c | 25 ++++++++++++------------- 6 files changed, 19 insertions(+), 49 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b1201d25a8a4..a6002084f09c 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -207,6 +207,7 @@ config ARM64 select HAVE_IOREMAP_PROT select HAVE_IRQ_TIME_ACCOUNTING select HAVE_KVM + select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI select HAVE_PERF_EVENTS select HAVE_PERF_REGS @@ -577,7 +578,6 @@ config ARM64_ERRATUM_845719 config ARM64_ERRATUM_843419 bool "Cortex-A53: 843419: A load or store might access an incorrect address" default y - select ARM64_MODULE_PLTS if MODULES help This option links the kernel with '--fix-cortex-a53-843419' and enables PLT support to replace certain ADRP instructions, which can @@ -2107,26 +2107,6 @@ config ARM64_SME register state capable of holding two dimensional matrix tiles to enable various matrix operations. -config ARM64_MODULE_PLTS - bool "Use PLTs to allow module memory to spill over into vmalloc area" - depends on MODULES - select HAVE_MOD_ARCH_SPECIFIC - help - Allocate PLTs when loading modules so that jumps and calls whose - targets are too far away for their relative offsets to be encoded - in the instructions themselves can be bounced via veneers in the - module's PLT. This allows modules to be allocated in the generic - vmalloc area after the dedicated module memory area has been - exhausted. - - When running with address space randomization (KASLR), the module - region itself may be too far away for ordinary relative jumps and - calls, and so in that case, module PLTs are required and cannot be - disabled. - - Specific errata workaround(s) might also force module PLTs to be - enabled (ARM64_ERRATUM_843419). - config ARM64_PSEUDO_NMI bool "Support for NMI-like interrupts" select ARM_GIC_V3 @@ -2167,7 +2147,6 @@ config RELOCATABLE config RANDOMIZE_BASE bool "Randomize the address of the kernel image" - select ARM64_MODULE_PLTS if MODULES select RELOCATABLE help Randomizes the virtual address at which the kernel image is @@ -2198,9 +2177,8 @@ config RANDOMIZE_MODULE_REGION_FULL When this option is not set, the module region will be randomized over a limited range that contains the [_stext, _etext] interval of the core kernel, so branch relocations are almost always in range unless - ARM64_MODULE_PLTS is enabled and the region is exhausted. In this - particular case of region exhaustion, modules might be able to fall - back to a larger 2GB area. + the region is exhausted. In this particular case of region + exhaustion, modules might be able to fall back to a larger 2GB area. config CC_HAVE_STACKPROTECTOR_SYSREG def_bool $(cc-option,-mstack-protector-guard=sysreg -mstack-protector-guard-reg=sp_el0 -mstack-protector-guard-offset=0) diff --git a/arch/arm64/include/asm/module.h b/arch/arm64/include/asm/module.h index 3e7dcc5fa2f2..bfa6638b4c93 100644 --- a/arch/arm64/include/asm/module.h +++ b/arch/arm64/include/asm/module.h @@ -7,7 +7,6 @@ #include -#ifdef CONFIG_ARM64_MODULE_PLTS struct mod_plt_sec { int plt_shndx; int plt_num_entries; @@ -21,7 +20,6 @@ struct mod_arch_specific { /* for CONFIG_DYNAMIC_FTRACE */ struct plt_entry *ftrace_trampolines; }; -#endif u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs, void *loc, const Elf64_Rela *rela, diff --git a/arch/arm64/include/asm/module.lds.h b/arch/arm64/include/asm/module.lds.h index dbba4b7559aa..b9ae8349e35d 100644 --- a/arch/arm64/include/asm/module.lds.h +++ b/arch/arm64/include/asm/module.lds.h @@ -1,9 +1,7 @@ SECTIONS { -#ifdef CONFIG_ARM64_MODULE_PLTS .plt 0 : { BYTE(0) } .init.plt 0 : { BYTE(0) } .text.ftrace_trampoline 0 : { BYTE(0) } -#endif #ifdef CONFIG_KASAN_SW_TAGS /* diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 7c2bb4e72476..3864a64e2b2b 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -42,8 +42,7 @@ obj-$(CONFIG_COMPAT) += sigreturn32.o obj-$(CONFIG_COMPAT_ALIGNMENT_FIXUPS) += compat_alignment.o obj-$(CONFIG_KUSER_HELPERS) += kuser32.o obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o -obj-$(CONFIG_MODULES) += module.o -obj-$(CONFIG_ARM64_MODULE_PLTS) += module-plts.o +obj-$(CONFIG_MODULES) += module.o module-plts.o obj-$(CONFIG_PERF_EVENTS) += perf_regs.o perf_callchain.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o obj-$(CONFIG_CPU_PM) += sleep.o suspend.o diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index 432626c866a8..a650f5e11fc5 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -197,7 +197,7 @@ int ftrace_update_ftrace_func(ftrace_func_t func) static struct plt_entry *get_ftrace_plt(struct module *mod) { -#ifdef CONFIG_ARM64_MODULE_PLTS +#ifdef CONFIG_MODULES struct plt_entry *plt = mod->arch.ftrace_trampolines; return &plt[FTRACE_PLT_IDX]; @@ -249,7 +249,7 @@ static bool ftrace_find_callable_addr(struct dyn_ftrace *rec, * must use a PLT to reach it. We can only place PLTs for modules, and * only when module PLT support is built-in. */ - if (!IS_ENABLED(CONFIG_ARM64_MODULE_PLTS)) + if (!IS_ENABLED(CONFIG_MODULES)) return false; /* @@ -431,10 +431,8 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, * * Note: 'mod' is only set at module load time. */ - if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_ARGS) && - IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && mod) { + if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_ARGS) && mod) return aarch64_insn_patch_text_nosync((void *)pc, new); - } if (!ftrace_find_callable_addr(rec, mod, &addr)) return -EINVAL; diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index 51fd8abd9b74..f64636c2fd05 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -73,25 +73,26 @@ subsys_initcall(kaslr_module_init) void *module_alloc(unsigned long size) { u64 module_alloc_end = module_alloc_base + MODULES_VSIZE; - gfp_t gfp_mask = GFP_KERNEL; void *p; - /* Silence the initial allocation */ - if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS)) - gfp_mask |= __GFP_NOWARN; - + /* + * Where possible, prefer to allocate within direct branch range of the + * kernel such that no PLTs are necessary. This may fail, so we pass + * __GFP_NOWARN to silence the resulting warning. + */ p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, - module_alloc_end, gfp_mask, PAGE_KERNEL, 0, - NUMA_NO_NODE, __builtin_return_address(0)); + module_alloc_end, GFP_KERNEL | __GFP_NOWARN, + PAGE_KERNEL, 0, NUMA_NO_NODE, + __builtin_return_address(0)); - if (!p && IS_ENABLED(CONFIG_ARM64_MODULE_PLTS)) { + if (!p) { p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, module_alloc_base + SZ_2G, GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); } - if (p && (kasan_alloc_module_shadow(p, size, gfp_mask) < 0)) { + if (p && (kasan_alloc_module_shadow(p, size, GFP_KERNEL) < 0)) { vfree(p); return NULL; } @@ -479,9 +480,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs, case R_AARCH64_CALL26: ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26, AARCH64_INSN_IMM_26); - - if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && - ovf == -ERANGE) { + if (ovf == -ERANGE) { val = module_emit_plt_entry(me, sechdrs, loc, &rel[i], sym); if (!val) return -ENOEXEC; @@ -518,7 +517,7 @@ static int module_init_ftrace_plt(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod) { -#if defined(CONFIG_ARM64_MODULE_PLTS) && defined(CONFIG_DYNAMIC_FTRACE) +#if defined(CONFIG_DYNAMIC_FTRACE) const Elf_Shdr *s; struct plt_entry *plts; -- cgit From 3e35d303ab7d22c4b6597e56ba46ee7cc61f3a5a Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 30 May 2023 12:03:28 +0100 Subject: arm64: module: rework module VA range selection Currently, the modules region is 128M in size, which is a problem for some large modules. Shanker reports [1] that the NVIDIA GPU driver alone can consume 110M of module space in some configurations. We'd like to make the modules region a full 2G such that we can always make use of a 2G range. It's possible to build kernel images which are larger than 128M in some configurations, such as when many debug options are selected and many drivers are built in. In these configurations, we can't legitimately select a base for a 128M module region, though we currently select a value for which allocation will fail. It would be nicer to have a diagnostic message in this case. Similarly, in theory it's possible to build a kernel image which is larger than 2G and which cannot support modules. While this isn't likely to be the case for any realistic kernel deplyed in the field, it would be nice if we could print a diagnostic in this case. This patch reworks the module VA range selection to use a 2G range, and improves handling of cases where we cannot select legitimate module regions. We now attempt to select a 128M region and a 2G region: * The 128M region is selected such that modules can use direct branches (with JUMP26/CALL26 relocations) to branch to kernel code and other modules, and so that modules can reference data and text (using PREL32 relocations) anywhere in the kernel image and other modules. This region covers the entire kernel image (rather than just the text) to ensure that all PREL32 relocations are in range even when the kernel data section is absurdly large. Where we cannot allocate from this region, we'll fall back to the full 2G region. * The 2G region is selected such that modules can use direct branches with PLTs to branch to kernel code and other modules, and so that modules can use reference data and text (with PREL32 relocations) in the kernel image and other modules. This region covers the entire kernel image, and the 128M region (if one is selected). The two module regions are randomized independently while ensuring the constraints described above. [1] https://lore.kernel.org/linux-arm-kernel/159ceeab-09af-3174-5058-445bc8dcf85b@nvidia.com/ Signed-off-by: Mark Rutland Reviewed-by: Ard Biesheuvel Cc: Shanker Donthineni Cc: Will Deacon Tested-by: Shanker Donthineni Link: https://lore.kernel.org/r/20230530110328.2213762-7-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- Documentation/arm64/memory.rst | 8 +-- arch/arm64/include/asm/memory.h | 2 +- arch/arm64/kernel/module.c | 139 +++++++++++++++++++++++++++------------- 3 files changed, 99 insertions(+), 50 deletions(-) diff --git a/Documentation/arm64/memory.rst b/Documentation/arm64/memory.rst index 2a641ba7be3b..55a55f30eed8 100644 --- a/Documentation/arm64/memory.rst +++ b/Documentation/arm64/memory.rst @@ -33,8 +33,8 @@ AArch64 Linux memory layout with 4KB pages + 4 levels (48-bit):: 0000000000000000 0000ffffffffffff 256TB user ffff000000000000 ffff7fffffffffff 128TB kernel logical memory map [ffff600000000000 ffff7fffffffffff] 32TB [kasan shadow region] - ffff800000000000 ffff800007ffffff 128MB modules - ffff800008000000 fffffbffefffffff 124TB vmalloc + ffff800000000000 ffff80007fffffff 2GB modules + ffff800080000000 fffffbffefffffff 124TB vmalloc fffffbfff0000000 fffffbfffdffffff 224MB fixed mappings (top down) fffffbfffe000000 fffffbfffe7fffff 8MB [guard region] fffffbfffe800000 fffffbffff7fffff 16MB PCI I/O space @@ -50,8 +50,8 @@ AArch64 Linux memory layout with 64KB pages + 3 levels (52-bit with HW support): 0000000000000000 000fffffffffffff 4PB user fff0000000000000 ffff7fffffffffff ~4PB kernel logical memory map [fffd800000000000 ffff7fffffffffff] 512TB [kasan shadow region] - ffff800000000000 ffff800007ffffff 128MB modules - ffff800008000000 fffffbffefffffff 124TB vmalloc + ffff800000000000 ffff80007fffffff 2GB modules + ffff800080000000 fffffbffefffffff 124TB vmalloc fffffbfff0000000 fffffbfffdffffff 224MB fixed mappings (top down) fffffbfffe000000 fffffbfffe7fffff 8MB [guard region] fffffbfffe800000 fffffbffff7fffff 16MB PCI I/O space diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index 215efc3bbbcf..6e0e5722f229 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -46,7 +46,7 @@ #define KIMAGE_VADDR (MODULES_END) #define MODULES_END (MODULES_VADDR + MODULES_VSIZE) #define MODULES_VADDR (_PAGE_END(VA_BITS_MIN)) -#define MODULES_VSIZE (SZ_128M) +#define MODULES_VSIZE (SZ_2G) #define VMEMMAP_START (-(UL(1) << (VA_BITS - VMEMMAP_SHIFT))) #define VMEMMAP_END (VMEMMAP_START + VMEMMAP_SIZE) #define PCI_IO_END (VMEMMAP_START - SZ_8M) diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index f64636c2fd05..dd851297596e 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -7,6 +7,8 @@ * Author: Will Deacon */ +#define pr_fmt(fmt) "Modules: " fmt + #include #include #include @@ -24,72 +26,119 @@ #include #include -static u64 __ro_after_init module_alloc_base = (u64)_etext - MODULES_VSIZE; +static u64 module_direct_base __ro_after_init = 0; +static u64 module_plt_base __ro_after_init = 0; -#ifdef CONFIG_RANDOMIZE_BASE -static int __init kaslr_module_init(void) +/* + * Choose a random page-aligned base address for a window of 'size' bytes which + * entirely contains the interval [start, end - 1]. + */ +static u64 __init random_bounding_box(u64 size, u64 start, u64 end) { - u64 module_range; - u32 seed; + u64 max_pgoff, pgoff; - if (!kaslr_enabled()) + if ((end - start) >= size) return 0; - seed = get_random_u32(); + max_pgoff = (size - (end - start)) / PAGE_SIZE; + pgoff = get_random_u32_inclusive(0, max_pgoff); - if (IS_ENABLED(CONFIG_RANDOMIZE_MODULE_REGION_FULL)) { - /* - * Randomize the module region over a 2 GB window covering the - * kernel. This reduces the risk of modules leaking information - * about the address of the kernel itself, but results in - * branches between modules and the core kernel that are - * resolved via PLTs. (Branches between modules will be - * resolved normally.) - */ - module_range = SZ_2G - (u64)(_end - _stext); - module_alloc_base = max((u64)_end - SZ_2G, (u64)MODULES_VADDR); + return start - pgoff * PAGE_SIZE; +} + +/* + * Modules may directly reference data and text anywhere within the kernel + * image and other modules. References using PREL32 relocations have a +/-2G + * range, and so we need to ensure that the entire kernel image and all modules + * fall within a 2G window such that these are always within range. + * + * Modules may directly branch to functions and code within the kernel text, + * and to functions and code within other modules. These branches will use + * CALL26/JUMP26 relocations with a +/-128M range. Without PLTs, we must ensure + * that the entire kernel text and all module text falls within a 128M window + * such that these are always within range. With PLTs, we can expand this to a + * 2G window. + * + * We chose the 128M region to surround the entire kernel image (rather than + * just the text) as using the same bounds for the 128M and 2G regions ensures + * by construction that we never select a 128M region that is not a subset of + * the 2G region. For very large and unusual kernel configurations this means + * we may fall back to PLTs where they could have been avoided, but this keeps + * the logic significantly simpler. + */ +static int __init module_init_limits(void) +{ + u64 kernel_end = (u64)_end; + u64 kernel_start = (u64)_text; + u64 kernel_size = kernel_end - kernel_start; + + /* + * The default modules region is placed immediately below the kernel + * image, and is large enough to use the full 2G relocation range. + */ + BUILD_BUG_ON(KIMAGE_VADDR != MODULES_END); + BUILD_BUG_ON(MODULES_VSIZE < SZ_2G); + + if (!kaslr_enabled()) { + if (kernel_size < SZ_128M) + module_direct_base = kernel_end - SZ_128M; + if (kernel_size < SZ_2G) + module_plt_base = kernel_end - SZ_2G; } else { - /* - * Randomize the module region by setting module_alloc_base to - * a PAGE_SIZE multiple in the range [_etext - MODULES_VSIZE, - * _stext) . This guarantees that the resulting region still - * covers [_stext, _etext], and that all relative branches can - * be resolved without veneers unless this region is exhausted - * and we fall back to a larger 2GB window in module_alloc() - * when ARM64_MODULE_PLTS is enabled. - */ - module_range = MODULES_VSIZE - (u64)(_etext - _stext); + u64 min = kernel_start; + u64 max = kernel_end; + + if (IS_ENABLED(CONFIG_RANDOMIZE_MODULE_REGION_FULL)) { + pr_info("2G module region forced by RANDOMIZE_MODULE_REGION_FULL\n"); + } else { + module_direct_base = random_bounding_box(SZ_128M, min, max); + if (module_direct_base) { + min = module_direct_base; + max = module_direct_base + SZ_128M; + } + } + + module_plt_base = random_bounding_box(SZ_2G, min, max); } - /* use the lower 21 bits to randomize the base of the module region */ - module_alloc_base += (module_range * (seed & ((1 << 21) - 1))) >> 21; - module_alloc_base &= PAGE_MASK; + pr_info("%llu pages in range for non-PLT usage", + module_direct_base ? (SZ_128M - kernel_size) / PAGE_SIZE : 0); + pr_info("%llu pages in range for PLT usage", + module_plt_base ? (SZ_2G - kernel_size) / PAGE_SIZE : 0); return 0; } -subsys_initcall(kaslr_module_init) -#endif +subsys_initcall(module_init_limits); void *module_alloc(unsigned long size) { - u64 module_alloc_end = module_alloc_base + MODULES_VSIZE; - void *p; + void *p = NULL; /* * Where possible, prefer to allocate within direct branch range of the - * kernel such that no PLTs are necessary. This may fail, so we pass - * __GFP_NOWARN to silence the resulting warning. + * kernel such that no PLTs are necessary. */ - p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, - module_alloc_end, GFP_KERNEL | __GFP_NOWARN, - PAGE_KERNEL, 0, NUMA_NO_NODE, - __builtin_return_address(0)); + if (module_direct_base) { + p = __vmalloc_node_range(size, MODULE_ALIGN, + module_direct_base, + module_direct_base + SZ_128M, + GFP_KERNEL | __GFP_NOWARN, + PAGE_KERNEL, 0, NUMA_NO_NODE, + __builtin_return_address(0)); + } + + if (!p && module_plt_base) { + p = __vmalloc_node_range(size, MODULE_ALIGN, + module_plt_base, + module_plt_base + SZ_2G, + GFP_KERNEL | __GFP_NOWARN, + PAGE_KERNEL, 0, NUMA_NO_NODE, + __builtin_return_address(0)); + } if (!p) { - p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, - module_alloc_base + SZ_2G, GFP_KERNEL, - PAGE_KERNEL, 0, NUMA_NO_NODE, - __builtin_return_address(0)); + pr_warn_ratelimited("%s: unable to allocate memory\n", + __func__); } if (p && (kasan_alloc_module_shadow(p, size, GFP_KERNEL) < 0)) { -- cgit From 3def3387f7556efa5f7ec738a3511758e0a4b1d1 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 23 May 2023 19:36:59 +0100 Subject: arm64/sysreg: Convert MDCCINT_EL1 to automatic register generation Convert MDCCINT_EL1 to automatic register generation as per DDI0616 2023-03. No functional change. Reviewed-by: Anshuman Khandual Reviewed-by: Shaoqin Huang Signed-off-by: Mark Brown Reviewed-by: Oliver Upton Link: https://lore.kernel.org/r/20230419-arm64-syreg-gen-v2-1-4c6add1f6257@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 1 - arch/arm64/tools/sysreg | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index e72d9aaab6b1..3d69bda0e608 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -135,7 +135,6 @@ #define SYS_SVCR_SMSTOP_SMZA_EL0 sys_reg(0, 3, 4, 6, 3) #define SYS_OSDTRRX_EL1 sys_reg(2, 0, 0, 0, 2) -#define SYS_MDCCINT_EL1 sys_reg(2, 0, 0, 2, 0) #define SYS_MDSCR_EL1 sys_reg(2, 0, 0, 2, 2) #define SYS_OSDTRTX_EL1 sys_reg(2, 0, 0, 3, 2) #define SYS_OSECCR_EL1 sys_reg(2, 0, 0, 6, 2) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index c9a0d1fa3209..df7a7ba97b43 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -48,6 +48,13 @@ # feature that introduces them (eg, FEAT_LS64_ACCDATA introduces enumeration # item ACCDATA) though it may be more taseful to do something else. +Sysreg MDCCINT_EL1 2 0 0 2 0 +Res0 63:31 +Field 30 RX +Field 29 TX +Res0 28:0 +EndSysreg + Sysreg ID_PFR0_EL1 3 0 0 1 0 Res0 63:32 UnsignedEnum 31:28 RAS -- cgit From 103b88427bc514c5f7d2d28b9f4e10cddc8900e0 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 23 May 2023 19:37:00 +0100 Subject: arm64/sysreg: Convert MDSCR_EL1 to automatic register generation Convert MDSCR_EL1 to automatic register generation as per DDI0616 2023-03. No functional change. Reviewed-by: Shaoqin Huang Signed-off-by: Mark Brown Reviewed-by: Oliver Upton Link: https://lore.kernel.org/r/20230419-arm64-syreg-gen-v2-2-4c6add1f6257@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 1 - arch/arm64/tools/sysreg | 28 ++++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 3d69bda0e608..95de1aaee0e9 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -135,7 +135,6 @@ #define SYS_SVCR_SMSTOP_SMZA_EL0 sys_reg(0, 3, 4, 6, 3) #define SYS_OSDTRRX_EL1 sys_reg(2, 0, 0, 0, 2) -#define SYS_MDSCR_EL1 sys_reg(2, 0, 0, 2, 2) #define SYS_OSDTRTX_EL1 sys_reg(2, 0, 0, 3, 2) #define SYS_OSECCR_EL1 sys_reg(2, 0, 0, 6, 2) #define SYS_DBGBVRn_EL1(n) sys_reg(2, 0, 0, n, 4) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index df7a7ba97b43..601cc8024734 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -55,6 +55,34 @@ Field 29 TX Res0 28:0 EndSysreg +Sysreg MDSCR_EL1 2 0 0 2 2 +Res0 63:36 +Field 35 EHBWE +Field 34 EnSPM +Field 33 TTA +Field 32 EMBWE +Field 31 TFO +Field 30 RXfull +Field 29 TXfull +Res0 28 +Field 27 RXO +Field 26 TXU +Res0 25:24 +Field 23:22 INTdis +Field 21 TDA +Res0 20 +Field 19 SC2 +Res0 18:16 +Field 15 MDE +Field 14 HDE +Field 13 KDE +Field 12 TDCC +Res0 11:7 +Field 6 ERR +Res0 5:1 +Field 0 SS +EndSysreg + Sysreg ID_PFR0_EL1 3 0 0 1 0 Res0 63:32 UnsignedEnum 31:28 RAS -- cgit From 187de7c2aad86c7bde6f695bfadb0ecd489de55c Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 23 May 2023 19:37:01 +0100 Subject: arm64/sysreg: Standardise naming of bitfield constants in OSL[AS]R_EL1 Our standard scheme for naming the constants for bitfields in system registers includes _ELx in the name but not the SYS_, update the constants for OSL[AS]R_EL1 to follow this convention. Reviewed-by: Shaoqin Huang Signed-off-by: Mark Brown Reviewed-by: Oliver Upton Link: https://lore.kernel.org/r/20230419-arm64-syreg-gen-v2-3-4c6add1f6257@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/kvm_host.h | 2 +- arch/arm64/include/asm/sysreg.h | 10 +++++----- arch/arm64/kvm/sys_regs.c | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 7e7e19ef6993..e759e6b0cd02 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1031,7 +1031,7 @@ void kvm_arm_clear_debug(struct kvm_vcpu *vcpu); void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu); #define kvm_vcpu_os_lock_enabled(vcpu) \ - (!!(__vcpu_sys_reg(vcpu, OSLSR_EL1) & SYS_OSLSR_OSLK)) + (!!(__vcpu_sys_reg(vcpu, OSLSR_EL1) & OSLSR_EL1_OSLK)) int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 95de1aaee0e9..979975c8be2c 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -144,13 +144,13 @@ #define SYS_MDRAR_EL1 sys_reg(2, 0, 1, 0, 0) #define SYS_OSLAR_EL1 sys_reg(2, 0, 1, 0, 4) -#define SYS_OSLAR_OSLK BIT(0) +#define OSLAR_EL1_OSLK BIT(0) #define SYS_OSLSR_EL1 sys_reg(2, 0, 1, 1, 4) -#define SYS_OSLSR_OSLM_MASK (BIT(3) | BIT(0)) -#define SYS_OSLSR_OSLM_NI 0 -#define SYS_OSLSR_OSLM_IMPLEMENTED BIT(3) -#define SYS_OSLSR_OSLK BIT(1) +#define OSLSR_EL1_OSLM_MASK (BIT(3) | BIT(0)) +#define OSLSR_EL1_OSLM_NI 0 +#define OSLSR_EL1_OSLM_IMPLEMENTED BIT(3) +#define OSLSR_EL1_OSLK BIT(1) #define SYS_OSDLR_EL1 sys_reg(2, 0, 1, 3, 4) #define SYS_DBGPRCR_EL1 sys_reg(2, 0, 1, 4, 4) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 71b12094d613..e834d8897843 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -388,9 +388,9 @@ static bool trap_oslar_el1(struct kvm_vcpu *vcpu, return read_from_write_only(vcpu, p, r); /* Forward the OSLK bit to OSLSR */ - oslsr = __vcpu_sys_reg(vcpu, OSLSR_EL1) & ~SYS_OSLSR_OSLK; - if (p->regval & SYS_OSLAR_OSLK) - oslsr |= SYS_OSLSR_OSLK; + oslsr = __vcpu_sys_reg(vcpu, OSLSR_EL1) & ~OSLSR_EL1_OSLK; + if (p->regval & OSLAR_EL1_OSLK) + oslsr |= OSLSR_EL1_OSLK; __vcpu_sys_reg(vcpu, OSLSR_EL1) = oslsr; return true; @@ -414,7 +414,7 @@ static int set_oslsr_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, * The only modifiable bit is the OSLK bit. Refuse the write if * userspace attempts to change any other bit in the register. */ - if ((val ^ rd->val) & ~SYS_OSLSR_OSLK) + if ((val ^ rd->val) & ~OSLSR_EL1_OSLK) return -EINVAL; __vcpu_sys_reg(vcpu, rd->reg) = val; @@ -1781,7 +1781,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_MDRAR_EL1), trap_raz_wi }, { SYS_DESC(SYS_OSLAR_EL1), trap_oslar_el1 }, { SYS_DESC(SYS_OSLSR_EL1), trap_oslsr_el1, reset_val, OSLSR_EL1, - SYS_OSLSR_OSLM_IMPLEMENTED, .set_user = set_oslsr_el1, }, + OSLSR_EL1_OSLM_IMPLEMENTED, .set_user = set_oslsr_el1, }, { SYS_DESC(SYS_OSDLR_EL1), trap_raz_wi }, { SYS_DESC(SYS_DBGPRCR_EL1), trap_raz_wi }, { SYS_DESC(SYS_DBGCLAIMSET_EL1), trap_raz_wi }, -- cgit From 31d504fce595f2aab00889e05bf8cea222e65d90 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 23 May 2023 19:37:02 +0100 Subject: arm64/sysreg: Convert OSLAR_EL1 to automatic generation Convert OSLAR_EL1 to automatic generation as per DDI0601 2023-03. No functional change. Reviewed-by: Shaoqin Huang Signed-off-by: Mark Brown Reviewed-by: Oliver Upton Link: https://lore.kernel.org/r/20230419-arm64-syreg-gen-v2-4-4c6add1f6257@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 3 --- arch/arm64/tools/sysreg | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 979975c8be2c..1901b676d7c6 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -143,9 +143,6 @@ #define SYS_DBGWCRn_EL1(n) sys_reg(2, 0, 0, n, 7) #define SYS_MDRAR_EL1 sys_reg(2, 0, 1, 0, 0) -#define SYS_OSLAR_EL1 sys_reg(2, 0, 1, 0, 4) -#define OSLAR_EL1_OSLK BIT(0) - #define SYS_OSLSR_EL1 sys_reg(2, 0, 1, 1, 4) #define OSLSR_EL1_OSLM_MASK (BIT(3) | BIT(0)) #define OSLSR_EL1_OSLM_NI 0 diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 601cc8024734..b0aefdf9ed34 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -83,6 +83,11 @@ Res0 5:1 Field 0 SS EndSysreg +Sysreg OSLAR_EL1 2 0 1 0 4 +Res0 63:1 +Field 0 OSLK +EndSysreg + Sysreg ID_PFR0_EL1 3 0 0 1 0 Res0 63:32 UnsignedEnum 31:28 RAS -- cgit From 7b416a16222901082d9c0c7748d5b446a2d85d90 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 23 May 2023 19:37:03 +0100 Subject: arm64/sysreg: Convert OSDTRRX_EL1 to automatic generation Convert OSDTRRX_EL1 to automatic generation as per DDI0601 2023-03, no functional changes. Signed-off-by: Mark Brown Reviewed-by: Shaoqin Huang Reviewed-by: Oliver Upton Link: https://lore.kernel.org/r/20230419-arm64-syreg-gen-v2-5-4c6add1f6257@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 1 - arch/arm64/tools/sysreg | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 1901b676d7c6..da954a6eba24 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -134,7 +134,6 @@ #define SYS_SVCR_SMSTART_SM_EL0 sys_reg(0, 3, 4, 3, 3) #define SYS_SVCR_SMSTOP_SMZA_EL0 sys_reg(0, 3, 4, 6, 3) -#define SYS_OSDTRRX_EL1 sys_reg(2, 0, 0, 0, 2) #define SYS_OSDTRTX_EL1 sys_reg(2, 0, 0, 3, 2) #define SYS_OSECCR_EL1 sys_reg(2, 0, 0, 6, 2) #define SYS_DBGBVRn_EL1(n) sys_reg(2, 0, 0, n, 4) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index b0aefdf9ed34..e4653248dcac 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -48,6 +48,11 @@ # feature that introduces them (eg, FEAT_LS64_ACCDATA introduces enumeration # item ACCDATA) though it may be more taseful to do something else. +Sysreg OSDTRRX_EL1 2 0 0 0 2 +Res0 63:32 +Field 31:0 DTRRX +EndSysreg + Sysreg MDCCINT_EL1 2 0 0 2 0 Res0 63:31 Field 30 RX -- cgit From 42383388758a56ceda7cc7b7e941cd9f415511f3 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 23 May 2023 19:37:04 +0100 Subject: arm64/sysreg: Convert OSDTRTX_EL1 to automatic generation Convert OSDTRTX_EL1 to automatic generation as per DDI0601 2023-03. No functional changes. Signed-off-by: Mark Brown Reviewed-by: Shaoqin Huang Reviewed-by: Oliver Upton Link: https://lore.kernel.org/r/20230419-arm64-syreg-gen-v2-6-4c6add1f6257@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 1 - arch/arm64/tools/sysreg | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index da954a6eba24..d9711f1e47b2 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -134,7 +134,6 @@ #define SYS_SVCR_SMSTART_SM_EL0 sys_reg(0, 3, 4, 3, 3) #define SYS_SVCR_SMSTOP_SMZA_EL0 sys_reg(0, 3, 4, 6, 3) -#define SYS_OSDTRTX_EL1 sys_reg(2, 0, 0, 3, 2) #define SYS_OSECCR_EL1 sys_reg(2, 0, 0, 6, 2) #define SYS_DBGBVRn_EL1(n) sys_reg(2, 0, 0, n, 4) #define SYS_DBGBCRn_EL1(n) sys_reg(2, 0, 0, n, 5) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index e4653248dcac..e18ae1df41f4 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -88,6 +88,11 @@ Res0 5:1 Field 0 SS EndSysreg +Sysreg OSDTRTX_EL1 2 0 0 3 2 +Res0 63:32 +Field 31:0 DTRTX +EndSysreg + Sysreg OSLAR_EL1 2 0 1 0 4 Res0 63:1 Field 0 OSLK -- cgit From 175cea665877c89aae4bd449d67bd8f4d6f900a3 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 23 May 2023 19:37:05 +0100 Subject: arm64/sysreg: Convert OSECCR_EL1 to automatic generation Convert OSECCR_EL1 to automatic generation as per DDI0601 2023-03, no functional changes. Signed-off-by: Mark Brown Reviewed-by: Shaoqin Huang Reviewed-by: Oliver Upton Link: https://lore.kernel.org/r/20230419-arm64-syreg-gen-v2-7-4c6add1f6257@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 1 - arch/arm64/tools/sysreg | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index d9711f1e47b2..23a17da500a4 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -134,7 +134,6 @@ #define SYS_SVCR_SMSTART_SM_EL0 sys_reg(0, 3, 4, 3, 3) #define SYS_SVCR_SMSTOP_SMZA_EL0 sys_reg(0, 3, 4, 6, 3) -#define SYS_OSECCR_EL1 sys_reg(2, 0, 0, 6, 2) #define SYS_DBGBVRn_EL1(n) sys_reg(2, 0, 0, n, 4) #define SYS_DBGBCRn_EL1(n) sys_reg(2, 0, 0, n, 5) #define SYS_DBGWVRn_EL1(n) sys_reg(2, 0, 0, n, 6) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index e18ae1df41f4..41462785020b 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -93,6 +93,11 @@ Res0 63:32 Field 31:0 DTRTX EndSysreg +Sysreg OSECCR_EL1 2 0 0 6 2 +Res0 63:32 +Field 31:0 EDECCR +EndSysreg + Sysreg OSLAR_EL1 2 0 1 0 4 Res0 63:1 Field 0 OSLK -- cgit From 7f242982e408d530958dd6efee059f353f636cf1 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 7 Jun 2023 17:48:43 +0100 Subject: arm64: standardise cpucap bitmap names The 'cpu_hwcaps' and 'boot_capabilities' bitmaps are bitmaps have the same enumerated bits, but are named wildly differently for no good reason. The terms 'hwcaps' and 'capabilities' have become ambiguous over time (e.g. due to clashes with ELF hwcaps and the structures used to manage feature detection), and it would be nicer to use 'cpucaps', matching the header the enumerated bit indices are defined in. While this isn't a functional problem, it makes the code harder than necessary to understand, and hard to extend with related functionality (e.g. per-cpu cpucap bitmaps). To that end, this patch renames `boot_capabilities` to `boot_cpucaps` and `cpu_hwcaps` to `system_cpucaps`. This more clearly indicates the relationship between the two and aligns with terminology used elsewhere in our feature management code. This change was scripted with: | find . -type f -name '*.[chS]' -print0 | \ | xargs -0 sed -i 's/\/boot_cpucaps/' | find . -type f -name '*.[chS]' -print0 | \ | xargs -0 sed -i 's/\/system_cpucaps/' ... and the instance of "cpu_hwcap" (without a trailing "s") in corrected manually to "system_cpucaps". Subsequent patches will adjust the naming of related functions to better align with the `cpucap` naming. There should be no functional change as a result of this patch; this is purely a renaming exercise. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Suzuki K Poulose Cc: Marc Zyngier Cc: Will Deacon Link: https://lore.kernel.org/r/20230607164846.3967305-2-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/cpufeature.h | 12 ++++++------ arch/arm64/include/asm/mmu_context.h | 2 +- arch/arm64/kernel/alternative.c | 6 +++--- arch/arm64/kernel/cpufeature.c | 12 ++++++------ 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 6bf013fb110d..04e11cdeda14 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -107,7 +107,7 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; * CPU capabilities: * * We use arm64_cpu_capabilities to represent system features, errata work - * arounds (both used internally by kernel and tracked in cpu_hwcaps) and + * arounds (both used internally by kernel and tracked in system_cpucaps) and * ELF HWCAPs (which are exposed to user). * * To support systems with heterogeneous CPUs, we need to make sure that we @@ -419,12 +419,12 @@ static __always_inline bool is_hyp_code(void) return is_vhe_hyp_code() || is_nvhe_hyp_code(); } -extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); +extern DECLARE_BITMAP(system_cpucaps, ARM64_NCAPS); -extern DECLARE_BITMAP(boot_capabilities, ARM64_NCAPS); +extern DECLARE_BITMAP(boot_cpucaps, ARM64_NCAPS); #define for_each_available_cap(cap) \ - for_each_set_bit(cap, cpu_hwcaps, ARM64_NCAPS) + for_each_set_bit(cap, system_cpucaps, ARM64_NCAPS) bool this_cpu_has_cap(unsigned int cap); void cpu_set_feature(unsigned int num); @@ -449,7 +449,7 @@ static __always_inline bool cpus_have_cap(unsigned int num) { if (num >= ARM64_NCAPS) return false; - return arch_test_bit(num, cpu_hwcaps); + return arch_test_bit(num, system_cpucaps); } /* @@ -510,7 +510,7 @@ static inline void cpus_set_cap(unsigned int num) pr_warn("Attempt to set an illegal CPU capability (%d >= %d)\n", num, ARM64_NCAPS); } else { - __set_bit(num, cpu_hwcaps); + __set_bit(num, system_cpucaps); } } diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 56911691bef0..5e0402946c35 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -164,7 +164,7 @@ static inline void cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap) * up (i.e. cpufeature framework is not up yet) and * latter only when we enable CNP via cpufeature's * enable() callback. - * Also we rely on the cpu_hwcap bit being set before + * Also we rely on the system_cpucaps bit being set before * calling the enable() function. */ ttbr1 |= TTBR_CNP_BIT; diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c index d32d4ed5519b..51a5dbd7f324 100644 --- a/arch/arm64/kernel/alternative.c +++ b/arch/arm64/kernel/alternative.c @@ -192,7 +192,7 @@ static void __apply_alternatives(const struct alt_region *region, bitmap_or(applied_alternatives, applied_alternatives, feature_mask, ARM64_NCAPS); bitmap_and(applied_alternatives, applied_alternatives, - cpu_hwcaps, ARM64_NCAPS); + system_cpucaps, ARM64_NCAPS); } } @@ -239,7 +239,7 @@ static int __init __apply_alternatives_multi_stop(void *unused) } else { DECLARE_BITMAP(remaining_capabilities, ARM64_NCAPS); - bitmap_complement(remaining_capabilities, boot_capabilities, + bitmap_complement(remaining_capabilities, boot_cpucaps, ARM64_NCAPS); BUG_ON(all_alternatives_applied); @@ -274,7 +274,7 @@ void __init apply_boot_alternatives(void) pr_info("applying boot alternatives\n"); __apply_alternatives(&kernel_alternatives, false, - &boot_capabilities[0]); + &boot_cpucaps[0]); } #ifdef CONFIG_MODULES diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 7d7128c65161..68e012f4ea08 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -105,11 +105,11 @@ unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT; unsigned int compat_elf_hwcap2 __read_mostly; #endif -DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); -EXPORT_SYMBOL(cpu_hwcaps); +DECLARE_BITMAP(system_cpucaps, ARM64_NCAPS); +EXPORT_SYMBOL(system_cpucaps); static struct arm64_cpu_capabilities const __ro_after_init *cpu_hwcaps_ptrs[ARM64_NCAPS]; -DECLARE_BITMAP(boot_capabilities, ARM64_NCAPS); +DECLARE_BITMAP(boot_cpucaps, ARM64_NCAPS); bool arm64_use_ng_mappings = false; EXPORT_SYMBOL(arm64_use_ng_mappings); @@ -137,7 +137,7 @@ static cpumask_var_t cpu_32bit_el0_mask __cpumask_var_read_mostly; void dump_cpu_features(void) { /* file-wide pr_fmt adds "CPU features: " prefix */ - pr_emerg("0x%*pb\n", ARM64_NCAPS, &cpu_hwcaps); + pr_emerg("0x%*pb\n", ARM64_NCAPS, &system_cpucaps); } #define ARM64_CPUID_FIELDS(reg, field, min_value) \ @@ -2906,7 +2906,7 @@ static void update_cpu_capabilities(u16 scope_mask) cpus_set_cap(caps->capability); if ((scope_mask & SCOPE_BOOT_CPU) && (caps->type & SCOPE_BOOT_CPU)) - set_bit(caps->capability, boot_capabilities); + set_bit(caps->capability, boot_cpucaps); } } @@ -3207,7 +3207,7 @@ EXPORT_SYMBOL_GPL(this_cpu_has_cap); /* * This helper function is used in a narrow window when, * - The system wide safe registers are set with all the SMP CPUs and, - * - The SYSTEM_FEATURE cpu_hwcaps may not have been set. + * - The SYSTEM_FEATURE system_cpucaps may not have been set. * In all other cases cpus_have_{const_}cap() should be used. */ static bool __maybe_unused __system_matches_cap(unsigned int n) -- cgit From 5235c7e2cfccb5bbd71aeda26240bb7877072782 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 7 Jun 2023 17:48:44 +0100 Subject: arm64: alternatives: use cpucap naming To more clearly align the various users of the cpucap enumeration, this patch changes the alternative code to use the term `cpucap` in favour of `feature`. The alternative_has_feature_{likely,unlikely}() functions are renamed to alternative_has_cap_ Reviewed-by: Suzuki K Poulose Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20230607164846.3967305-3-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/alternative-macros.h | 54 ++++++++++++++--------------- arch/arm64/include/asm/alternative.h | 4 +-- arch/arm64/include/asm/cpufeature.h | 4 +-- arch/arm64/include/asm/irqflags.h | 2 +- arch/arm64/include/asm/lse.h | 2 +- arch/arm64/kernel/alternative.c | 17 +++++---- 6 files changed, 41 insertions(+), 42 deletions(-) diff --git a/arch/arm64/include/asm/alternative-macros.h b/arch/arm64/include/asm/alternative-macros.h index bdf1f6bcd010..94b486192e1f 100644 --- a/arch/arm64/include/asm/alternative-macros.h +++ b/arch/arm64/include/asm/alternative-macros.h @@ -23,17 +23,17 @@ #include -#define ALTINSTR_ENTRY(feature) \ +#define ALTINSTR_ENTRY(cpucap) \ " .word 661b - .\n" /* label */ \ " .word 663f - .\n" /* new instruction */ \ - " .hword " __stringify(feature) "\n" /* feature bit */ \ + " .hword " __stringify(cpucap) "\n" /* cpucap */ \ " .byte 662b-661b\n" /* source len */ \ " .byte 664f-663f\n" /* replacement len */ -#define ALTINSTR_ENTRY_CB(feature, cb) \ +#define ALTINSTR_ENTRY_CB(cpucap, cb) \ " .word 661b - .\n" /* label */ \ - " .word " __stringify(cb) "- .\n" /* callback */ \ - " .hword " __stringify(feature) "\n" /* feature bit */ \ + " .word " __stringify(cb) "- .\n" /* callback */ \ + " .hword " __stringify(cpucap) "\n" /* cpucap */ \ " .byte 662b-661b\n" /* source len */ \ " .byte 664f-663f\n" /* replacement len */ @@ -53,13 +53,13 @@ * * Alternatives with callbacks do not generate replacement instructions. */ -#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled) \ +#define __ALTERNATIVE_CFG(oldinstr, newinstr, cpucap, cfg_enabled) \ ".if "__stringify(cfg_enabled)" == 1\n" \ "661:\n\t" \ oldinstr "\n" \ "662:\n" \ ".pushsection .altinstructions,\"a\"\n" \ - ALTINSTR_ENTRY(feature) \ + ALTINSTR_ENTRY(cpucap) \ ".popsection\n" \ ".subsection 1\n" \ "663:\n\t" \ @@ -70,31 +70,31 @@ ".previous\n" \ ".endif\n" -#define __ALTERNATIVE_CFG_CB(oldinstr, feature, cfg_enabled, cb) \ +#define __ALTERNATIVE_CFG_CB(oldinstr, cpucap, cfg_enabled, cb) \ ".if "__stringify(cfg_enabled)" == 1\n" \ "661:\n\t" \ oldinstr "\n" \ "662:\n" \ ".pushsection .altinstructions,\"a\"\n" \ - ALTINSTR_ENTRY_CB(feature, cb) \ + ALTINSTR_ENTRY_CB(cpucap, cb) \ ".popsection\n" \ "663:\n\t" \ "664:\n\t" \ ".endif\n" -#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \ - __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg)) +#define _ALTERNATIVE_CFG(oldinstr, newinstr, cpucap, cfg, ...) \ + __ALTERNATIVE_CFG(oldinstr, newinstr, cpucap, IS_ENABLED(cfg)) -#define ALTERNATIVE_CB(oldinstr, feature, cb) \ - __ALTERNATIVE_CFG_CB(oldinstr, (1 << ARM64_CB_SHIFT) | (feature), 1, cb) +#define ALTERNATIVE_CB(oldinstr, cpucap, cb) \ + __ALTERNATIVE_CFG_CB(oldinstr, (1 << ARM64_CB_SHIFT) | (cpucap), 1, cb) #else #include -.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len +.macro altinstruction_entry orig_offset alt_offset cpucap orig_len alt_len .word \orig_offset - . .word \alt_offset - . - .hword (\feature) + .hword (\cpucap) .byte \orig_len .byte \alt_len .endm @@ -210,9 +210,9 @@ alternative_endif #endif /* __ASSEMBLY__ */ /* - * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature)); + * Usage: asm(ALTERNATIVE(oldinstr, newinstr, cpucap)); * - * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO)); + * Usage: asm(ALTERNATIVE(oldinstr, newinstr, cpucap, CONFIG_FOO)); * N.B. If CONFIG_FOO is specified, but not selected, the whole block * will be omitted, including oldinstr. */ @@ -224,15 +224,15 @@ alternative_endif #include static __always_inline bool -alternative_has_feature_likely(const unsigned long feature) +alternative_has_cap_likely(const unsigned long cpucap) { - compiletime_assert(feature < ARM64_NCAPS, - "feature must be < ARM64_NCAPS"); + compiletime_assert(cpucap < ARM64_NCAPS, + "cpucap must be < ARM64_NCAPS"); asm_volatile_goto( - ALTERNATIVE_CB("b %l[l_no]", %[feature], alt_cb_patch_nops) + ALTERNATIVE_CB("b %l[l_no]", %[cpucap], alt_cb_patch_nops) : - : [feature] "i" (feature) + : [cpucap] "i" (cpucap) : : l_no); @@ -242,15 +242,15 @@ l_no: } static __always_inline bool -alternative_has_feature_unlikely(const unsigned long feature) +alternative_has_cap_unlikely(const unsigned long cpucap) { - compiletime_assert(feature < ARM64_NCAPS, - "feature must be < ARM64_NCAPS"); + compiletime_assert(cpucap < ARM64_NCAPS, + "cpucap must be < ARM64_NCAPS"); asm_volatile_goto( - ALTERNATIVE("nop", "b %l[l_yes]", %[feature]) + ALTERNATIVE("nop", "b %l[l_yes]", %[cpucap]) : - : [feature] "i" (feature) + : [cpucap] "i" (cpucap) : : l_yes); diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h index a38b92e11811..fe657e878757 100644 --- a/arch/arm64/include/asm/alternative.h +++ b/arch/arm64/include/asm/alternative.h @@ -13,7 +13,7 @@ struct alt_instr { s32 orig_offset; /* offset to original instruction */ s32 alt_offset; /* offset to replacement instruction */ - u16 cpufeature; /* cpufeature bit set for replacement */ + u16 cpucap; /* cpucap bit set for replacement */ u8 orig_len; /* size of original instruction(s) */ u8 alt_len; /* size of new instruction(s), <= orig_len */ }; @@ -23,7 +23,7 @@ typedef void (*alternative_cb_t)(struct alt_instr *alt, void __init apply_boot_alternatives(void); void __init apply_alternatives_all(void); -bool alternative_is_applied(u16 cpufeature); +bool alternative_is_applied(u16 cpucap); #ifdef CONFIG_MODULES void apply_alternatives_module(void *start, size_t length); diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 04e11cdeda14..6de2a5b232fc 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -437,7 +437,7 @@ unsigned long cpu_get_elf_hwcap2(void); static __always_inline bool system_capabilities_finalized(void) { - return alternative_has_feature_likely(ARM64_ALWAYS_SYSTEM); + return alternative_has_cap_likely(ARM64_ALWAYS_SYSTEM); } /* @@ -464,7 +464,7 @@ static __always_inline bool __cpus_have_const_cap(int num) { if (num >= ARM64_NCAPS) return false; - return alternative_has_feature_unlikely(num); + return alternative_has_cap_unlikely(num); } /* diff --git a/arch/arm64/include/asm/irqflags.h b/arch/arm64/include/asm/irqflags.h index e0f5f6b73edd..1f31ec146d16 100644 --- a/arch/arm64/include/asm/irqflags.h +++ b/arch/arm64/include/asm/irqflags.h @@ -24,7 +24,7 @@ static __always_inline bool __irqflags_uses_pmr(void) { return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && - alternative_has_feature_unlikely(ARM64_HAS_GIC_PRIO_MASKING); + alternative_has_cap_unlikely(ARM64_HAS_GIC_PRIO_MASKING); } static __always_inline void __daif_local_irq_enable(void) diff --git a/arch/arm64/include/asm/lse.h b/arch/arm64/include/asm/lse.h index f99d74826a7e..cbbcdc35c4cd 100644 --- a/arch/arm64/include/asm/lse.h +++ b/arch/arm64/include/asm/lse.h @@ -18,7 +18,7 @@ static __always_inline bool system_uses_lse_atomics(void) { - return alternative_has_feature_likely(ARM64_HAS_LSE_ATOMICS); + return alternative_has_cap_likely(ARM64_HAS_LSE_ATOMICS); } #define __lse_ll_sc_body(op, ...) \ diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c index 51a5dbd7f324..53d13b2e5f59 100644 --- a/arch/arm64/kernel/alternative.c +++ b/arch/arm64/kernel/alternative.c @@ -24,8 +24,8 @@ #define ALT_ORIG_PTR(a) __ALT_PTR(a, orig_offset) #define ALT_REPL_PTR(a) __ALT_PTR(a, alt_offset) -#define ALT_CAP(a) ((a)->cpufeature & ~ARM64_CB_BIT) -#define ALT_HAS_CB(a) ((a)->cpufeature & ARM64_CB_BIT) +#define ALT_CAP(a) ((a)->cpucap & ~ARM64_CB_BIT) +#define ALT_HAS_CB(a) ((a)->cpucap & ARM64_CB_BIT) /* Volatile, as we may be patching the guts of READ_ONCE() */ static volatile int all_alternatives_applied; @@ -37,12 +37,12 @@ struct alt_region { struct alt_instr *end; }; -bool alternative_is_applied(u16 cpufeature) +bool alternative_is_applied(u16 cpucap) { - if (WARN_ON(cpufeature >= ARM64_NCAPS)) + if (WARN_ON(cpucap >= ARM64_NCAPS)) return false; - return test_bit(cpufeature, applied_alternatives); + return test_bit(cpucap, applied_alternatives); } /* @@ -141,7 +141,7 @@ static void clean_dcache_range_nopatch(u64 start, u64 end) static void __apply_alternatives(const struct alt_region *region, bool is_module, - unsigned long *feature_mask) + unsigned long *cpucap_mask) { struct alt_instr *alt; __le32 *origptr, *updptr; @@ -151,7 +151,7 @@ static void __apply_alternatives(const struct alt_region *region, int nr_inst; int cap = ALT_CAP(alt); - if (!test_bit(cap, feature_mask)) + if (!test_bit(cap, cpucap_mask)) continue; if (!cpus_have_cap(cap)) @@ -188,9 +188,8 @@ static void __apply_alternatives(const struct alt_region *region, icache_inval_all_pou(); isb(); - /* Ignore ARM64_CB bit from feature mask */ bitmap_or(applied_alternatives, applied_alternatives, - feature_mask, ARM64_NCAPS); + cpucap_mask, ARM64_NCAPS); bitmap_and(applied_alternatives, applied_alternatives, system_cpucaps, ARM64_NCAPS); } -- cgit From 1c8ae42975bd708f6c8c73d011025868b23076b3 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 7 Jun 2023 17:48:45 +0100 Subject: arm64: cpufeature: use cpucap naming To more clearly align the various users of the cpucap enumeration, this patch changes the cpufeature code to use the term `cpucap` in favour of `cpu_hwcap`. This more clearly aligns with other users of the cpucaps, and avoids confusion with the ELF hwcaps. There should be no functional change as a result of this patch; this is purely a renaming exercise. Signed-off-by: Mark Rutland Reviewed-by: Mark Brown Reviewed-by: Suzuki K Poulose Cc: Marc Zyngier Cc: Will Deacon Link: https://lore.kernel.org/r/20230607164846.3967305-4-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/kernel/cpufeature.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 68e012f4ea08..82e681952834 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -107,7 +107,7 @@ unsigned int compat_elf_hwcap2 __read_mostly; DECLARE_BITMAP(system_cpucaps, ARM64_NCAPS); EXPORT_SYMBOL(system_cpucaps); -static struct arm64_cpu_capabilities const __ro_after_init *cpu_hwcaps_ptrs[ARM64_NCAPS]; +static struct arm64_cpu_capabilities const __ro_after_init *cpucap_ptrs[ARM64_NCAPS]; DECLARE_BITMAP(boot_cpucaps, ARM64_NCAPS); @@ -954,24 +954,24 @@ extern const struct arm64_cpu_capabilities arm64_errata[]; static const struct arm64_cpu_capabilities arm64_features[]; static void __init -init_cpu_hwcaps_indirect_list_from_array(const struct arm64_cpu_capabilities *caps) +init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps) { for (; caps->matches; caps++) { if (WARN(caps->capability >= ARM64_NCAPS, "Invalid capability %d\n", caps->capability)) continue; - if (WARN(cpu_hwcaps_ptrs[caps->capability], + if (WARN(cpucap_ptrs[caps->capability], "Duplicate entry for capability %d\n", caps->capability)) continue; - cpu_hwcaps_ptrs[caps->capability] = caps; + cpucap_ptrs[caps->capability] = caps; } } -static void __init init_cpu_hwcaps_indirect_list(void) +static void __init init_cpucap_indirect_list(void) { - init_cpu_hwcaps_indirect_list_from_array(arm64_features); - init_cpu_hwcaps_indirect_list_from_array(arm64_errata); + init_cpucap_indirect_list_from_array(arm64_features); + init_cpucap_indirect_list_from_array(arm64_errata); } static void __init setup_boot_cpu_capabilities(void); @@ -1049,10 +1049,10 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) init_cpu_ftr_reg(SYS_GMID_EL1, info->reg_gmid); /* - * Initialize the indirect array of CPU hwcaps capabilities pointers - * before we handle the boot CPU below. + * Initialize the indirect array of CPU capabilities pointers before we + * handle the boot CPU below. */ - init_cpu_hwcaps_indirect_list(); + init_cpucap_indirect_list(); /* * Detect and enable early CPU capabilities based on the boot CPU, @@ -2048,9 +2048,9 @@ static bool has_address_auth_cpucap(const struct arm64_cpu_capabilities *entry, static bool has_address_auth_metacap(const struct arm64_cpu_capabilities *entry, int scope) { - bool api = has_address_auth_cpucap(cpu_hwcaps_ptrs[ARM64_HAS_ADDRESS_AUTH_IMP_DEF], scope); - bool apa = has_address_auth_cpucap(cpu_hwcaps_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA5], scope); - bool apa3 = has_address_auth_cpucap(cpu_hwcaps_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA3], scope); + bool api = has_address_auth_cpucap(cpucap_ptrs[ARM64_HAS_ADDRESS_AUTH_IMP_DEF], scope); + bool apa = has_address_auth_cpucap(cpucap_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA5], scope); + bool apa3 = has_address_auth_cpucap(cpucap_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA3], scope); return apa || apa3 || api; } @@ -2895,7 +2895,7 @@ static void update_cpu_capabilities(u16 scope_mask) scope_mask &= ARM64_CPUCAP_SCOPE_MASK; for (i = 0; i < ARM64_NCAPS; i++) { - caps = cpu_hwcaps_ptrs[i]; + caps = cpucap_ptrs[i]; if (!caps || !(caps->type & scope_mask) || cpus_have_cap(caps->capability) || !caps->matches(caps, cpucap_default_scope(caps))) @@ -2920,7 +2920,7 @@ static int cpu_enable_non_boot_scope_capabilities(void *__unused) u16 non_boot_scope = SCOPE_ALL & ~SCOPE_BOOT_CPU; for_each_available_cap(i) { - const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[i]; + const struct arm64_cpu_capabilities *cap = cpucap_ptrs[i]; if (WARN_ON(!cap)) continue; @@ -2950,7 +2950,7 @@ static void __init enable_cpu_capabilities(u16 scope_mask) for (i = 0; i < ARM64_NCAPS; i++) { unsigned int num; - caps = cpu_hwcaps_ptrs[i]; + caps = cpucap_ptrs[i]; if (!caps || !(caps->type & scope_mask)) continue; num = caps->capability; @@ -2995,7 +2995,7 @@ static void verify_local_cpu_caps(u16 scope_mask) scope_mask &= ARM64_CPUCAP_SCOPE_MASK; for (i = 0; i < ARM64_NCAPS; i++) { - caps = cpu_hwcaps_ptrs[i]; + caps = cpucap_ptrs[i]; if (!caps || !(caps->type & scope_mask)) continue; @@ -3194,7 +3194,7 @@ static void __init setup_boot_cpu_capabilities(void) bool this_cpu_has_cap(unsigned int n) { if (!WARN_ON(preemptible()) && n < ARM64_NCAPS) { - const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[n]; + const struct arm64_cpu_capabilities *cap = cpucap_ptrs[n]; if (cap) return cap->matches(cap, SCOPE_LOCAL_CPU); @@ -3213,7 +3213,7 @@ EXPORT_SYMBOL_GPL(this_cpu_has_cap); static bool __maybe_unused __system_matches_cap(unsigned int n) { if (n < ARM64_NCAPS) { - const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[n]; + const struct arm64_cpu_capabilities *cap = cpucap_ptrs[n]; if (cap) return cap->matches(cap, SCOPE_SYSTEM); -- cgit From 7dae5f086fceba6b0a4735b63dbcb4795f353c39 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 7 Jun 2023 17:48:46 +0100 Subject: arm64: cpufeature: fold cpus_set_cap() into update_cpu_capabilities() We only use cpus_set_cap() in update_cpu_capabilities(), where we open-code an analgous update to boot_cpucaps. Due to the way the cpucap_ptrs[] array is initialized, we know that the capability number cannot be greater than or equal to ARM64_NCAPS, so the warning is superfluous. Fold cpus_set_cap() into update_cpu_capabilities(), matching what we do for the boot_cpucaps, and making the relationship between the two a bit clearer. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Reviewed-by: Suzuki K Poulose Cc: Marc Zyngier Cc: Mark Brown Cc: Will Deacon Link: https://lore.kernel.org/r/20230607164846.3967305-5-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/cpufeature.h | 10 ---------- arch/arm64/kernel/cpufeature.c | 3 ++- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 6de2a5b232fc..7a95c324e52a 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -504,16 +504,6 @@ static __always_inline bool cpus_have_const_cap(int num) return cpus_have_cap(num); } -static inline void cpus_set_cap(unsigned int num) -{ - if (num >= ARM64_NCAPS) { - pr_warn("Attempt to set an illegal CPU capability (%d >= %d)\n", - num, ARM64_NCAPS); - } else { - __set_bit(num, system_cpucaps); - } -} - static inline int __attribute_const__ cpuid_feature_extract_signed_field_width(u64 features, int field, int width) { diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 82e681952834..bfadac361ac1 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2903,7 +2903,8 @@ static void update_cpu_capabilities(u16 scope_mask) if (caps->desc) pr_info("detected: %s\n", caps->desc); - cpus_set_cap(caps->capability); + + __set_bit(caps->capability, system_cpucaps); if ((scope_mask & SCOPE_BOOT_CPU) && (caps->type & SCOPE_BOOT_CPU)) set_bit(caps->capability, boot_cpucaps); -- cgit From 1da185fc8288fadb952e06881d0b75e924780103 Mon Sep 17 00:00:00 2001 From: Guo Hui Date: Fri, 26 May 2023 10:47:15 +0800 Subject: arm64: syscall: unmask DAIF for tracing status The following code: static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr, const syscall_fn_t syscall_table[]) { ... if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) { local_daif_mask(); flags = read_thread_flags(); -------------------------------- A if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP)) return; local_daif_restore(DAIF_PROCCTX); } trace_exit: syscall_trace_exit(regs); ------- B } 1. The flags in the if conditional statement should be used in the same way as the flags in the function syscall_trace_exit, because DAIF is not shielded in the function syscall_trace_exit, so when the flags are obtained in line A of the code, there is no need to shield DAIF. Don't care about the modification of flags after line A. 2. Masking DAIF caused syscall performance to deteriorate by about 10%. The Unixbench single core syscall performance data is as follows: Machine: Kunpeng 920 Mask DAIF: System Call Overhead 1172314.1 lps (10.0 s, 7 samples) System Benchmarks Partial Index BASELINE RESULT INDEX System Call Overhead 15000.0 1172314.1 781.5 ======== System Benchmarks Index Score (Partial Only) 781.5 Unmask DAIF: System Call Overhead 1287944.6 lps (10.0 s, 7 samples) System Benchmarks Partial Index BASELINE RESULT INDEX System Call Overhead 15000.0 1287944.6 858.6 ======== System Benchmarks Index Score (Partial Only) 858.6 Rationale from Mark Rutland as for why this is safe: This masking is an artifact of the old "ret_fast_syscall" assembly that was converted to C in commit: f37099b6992a0b81 ("arm64: convert syscall trace logic to C") The assembly would mask DAIF, check the thread flags, and fall through to kernel_exit without unmasking if no tracing was needed. The conversion copied this masking into the C version, though this wasn't strictly necessary. As (in general) thread flags can be manipulated by other threads, it's not safe to manipulate the thread flags with plain reads and writes, and since commit: 342b3808786518ce ("arm64: Snapshot thread flags") ... we use read_thread_flags() to read the flags atomically. With this, there is no need to mask DAIF transiently around reading the flags, as we only decide whether to trace while DAIF is masked, and the actual tracing occurs with DAIF unmasked. When el0_svc_common() returns its caller will unconditionally mask DAIF via exit_to_user_mode(), so the masking is redundant. Signed-off-by: Guo Hui Reviewed-by: Mark Rutland Link: https://lore.kernel.org/r/20230526024715.8773-1-guohui@uniontech.com [catalin.marinas@arm.com: updated comment with Mark's rationale] Signed-off-by: Catalin Marinas --- arch/arm64/kernel/syscall.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c index da84cf855c44..5a668d7f3c1f 100644 --- a/arch/arm64/kernel/syscall.c +++ b/arch/arm64/kernel/syscall.c @@ -147,11 +147,9 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr, * exit regardless, as the old entry assembly did. */ if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) { - local_daif_mask(); flags = read_thread_flags(); if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP)) return; - local_daif_restore(DAIF_PROCCTX); } trace_exit: -- cgit From fcea0ccf4fd7f5e0b978d3c18923eea4e431118d Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Tue, 6 Jun 2023 10:35:31 +0100 Subject: ACPI: bus: Consolidate all arm specific initialisation into acpi_arm_init() Move all of the ARM-specific initialization into one function namely acpi_arm_init(), so it is not necessary to modify/update bus.c every time a new piece of it is added. Cc: Lorenzo Pieralisi Cc: Rafael J. Wysocki Suggested-by: Rafael J. Wysocki Reviewed-by: Robin Murphy Reviewed-by: Hanjun Guo Link: https://lore.kernel.org/r/CAJZ5v0iBZRZmV_oU+VurqxnVMbFN_ttqrL=cLh0sUH+=u0PYsw@mail.gmail.com Signed-off-by: Sudeep Holla Reviewed-by: Lorenzo Pieralisi Acked-by: Rafael J. Wysocki Reviewed-by: Shaoqin Huang Link: https://lore.kernel.org/r/20230606093531.2746732-1-sudeep.holla@arm.com Signed-off-by: Catalin Marinas --- drivers/acpi/arm64/Makefile | 2 +- drivers/acpi/arm64/agdi.c | 2 +- drivers/acpi/arm64/apmt.c | 2 +- drivers/acpi/arm64/init.c | 13 +++++++++++++ drivers/acpi/arm64/init.h | 6 ++++++ drivers/acpi/arm64/iort.c | 1 + drivers/acpi/bus.c | 7 +------ include/linux/acpi.h | 6 ++++++ include/linux/acpi_agdi.h | 13 ------------- include/linux/acpi_apmt.h | 19 ------------------- include/linux/acpi_iort.h | 2 -- 11 files changed, 30 insertions(+), 43 deletions(-) create mode 100644 drivers/acpi/arm64/init.c create mode 100644 drivers/acpi/arm64/init.h delete mode 100644 include/linux/acpi_agdi.h delete mode 100644 include/linux/acpi_apmt.h diff --git a/drivers/acpi/arm64/Makefile b/drivers/acpi/arm64/Makefile index e21a9e84e394..f81fe24894b2 100644 --- a/drivers/acpi/arm64/Makefile +++ b/drivers/acpi/arm64/Makefile @@ -3,4 +3,4 @@ obj-$(CONFIG_ACPI_AGDI) += agdi.o obj-$(CONFIG_ACPI_IORT) += iort.o obj-$(CONFIG_ACPI_GTDT) += gtdt.o obj-$(CONFIG_ACPI_APMT) += apmt.o -obj-y += dma.o +obj-y += dma.o init.o diff --git a/drivers/acpi/arm64/agdi.c b/drivers/acpi/arm64/agdi.c index f605302395c3..8b3c7d42b41b 100644 --- a/drivers/acpi/arm64/agdi.c +++ b/drivers/acpi/arm64/agdi.c @@ -9,11 +9,11 @@ #define pr_fmt(fmt) "ACPI: AGDI: " fmt #include -#include #include #include #include #include +#include "init.h" struct agdi_data { int sdei_event; diff --git a/drivers/acpi/arm64/apmt.c b/drivers/acpi/arm64/apmt.c index 8cab69fa5d59..e5c3bc99fc79 100644 --- a/drivers/acpi/arm64/apmt.c +++ b/drivers/acpi/arm64/apmt.c @@ -10,10 +10,10 @@ #define pr_fmt(fmt) "ACPI: APMT: " fmt #include -#include #include #include #include +#include "init.h" #define DEV_NAME "arm-cs-arch-pmu" diff --git a/drivers/acpi/arm64/init.c b/drivers/acpi/arm64/init.c new file mode 100644 index 000000000000..d3ce53dda122 --- /dev/null +++ b/drivers/acpi/arm64/init.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include "init.h" + +void __init acpi_arm_init(void) +{ + if (IS_ENABLED(CONFIG_ACPI_AGDI)) + acpi_agdi_init(); + if (IS_ENABLED(CONFIG_ACPI_APMT)) + acpi_apmt_init(); + if (IS_ENABLED(CONFIG_ACPI_IORT)) + acpi_iort_init(); +} diff --git a/drivers/acpi/arm64/init.h b/drivers/acpi/arm64/init.h new file mode 100644 index 000000000000..a1715a2a34e9 --- /dev/null +++ b/drivers/acpi/arm64/init.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include + +void __init acpi_agdi_init(void); +void __init acpi_apmt_init(void); +void __init acpi_iort_init(void); diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 38fb84974f35..3631230a61c8 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -19,6 +19,7 @@ #include #include #include +#include "init.h" #define IORT_TYPE_MASK(type) (1 << (type)) #define IORT_MSI_TYPE (1 << ACPI_IORT_NODE_ITS_GROUP) diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index d161ff707de4..7a1eaf8c7bde 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -26,9 +26,6 @@ #include #include #endif -#include -#include -#include #include #include #include @@ -1408,7 +1405,7 @@ static int __init acpi_init(void) acpi_init_ffh(); pci_mmcfg_late_init(); - acpi_iort_init(); + acpi_arm_init(); acpi_viot_early_init(); acpi_hest_init(); acpi_ghes_init(); @@ -1420,8 +1417,6 @@ static int __init acpi_init(void) acpi_debugger_init(); acpi_setup_sb_notify_handler(); acpi_viot_init(); - acpi_agdi_init(); - acpi_apmt_init(); return 0; } diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 7b71dd74baeb..5ef126a0a50f 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1507,6 +1507,12 @@ static inline int find_acpi_cpu_topology_hetero_id(unsigned int cpu) } #endif +#ifdef CONFIG_ARM64 +void acpi_arm_init(void); +#else +static inline void acpi_arm_init(void) { } +#endif + #ifdef CONFIG_ACPI_PCC void acpi_init_pcc(void); #else diff --git a/include/linux/acpi_agdi.h b/include/linux/acpi_agdi.h deleted file mode 100644 index f477f0b452fa..000000000000 --- a/include/linux/acpi_agdi.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ - -#ifndef __ACPI_AGDI_H__ -#define __ACPI_AGDI_H__ - -#include - -#ifdef CONFIG_ACPI_AGDI -void __init acpi_agdi_init(void); -#else -static inline void acpi_agdi_init(void) {} -#endif -#endif /* __ACPI_AGDI_H__ */ diff --git a/include/linux/acpi_apmt.h b/include/linux/acpi_apmt.h deleted file mode 100644 index 40bd634d082f..000000000000 --- a/include/linux/acpi_apmt.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 - * - * ARM CoreSight PMU driver. - * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. - * - */ - -#ifndef __ACPI_APMT_H__ -#define __ACPI_APMT_H__ - -#include - -#ifdef CONFIG_ACPI_APMT -void acpi_apmt_init(void); -#else -static inline void acpi_apmt_init(void) { } -#endif /* CONFIG_ACPI_APMT */ - -#endif /* __ACPI_APMT_H__ */ diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h index b43be0987b19..e4e7bb6fa720 100644 --- a/include/linux/acpi_iort.h +++ b/include/linux/acpi_iort.h @@ -27,7 +27,6 @@ int iort_register_domain_token(int trans_id, phys_addr_t base, void iort_deregister_domain_token(int trans_id); struct fwnode_handle *iort_find_domain_token(int trans_id); #ifdef CONFIG_ACPI_IORT -void acpi_iort_init(void); u32 iort_msi_map_id(struct device *dev, u32 id); struct irq_domain *iort_get_device_domain(struct device *dev, u32 id, enum irq_domain_bus_token bus_token); @@ -43,7 +42,6 @@ int iort_iommu_configure_id(struct device *dev, const u32 *id_in); void iort_iommu_get_resv_regions(struct device *dev, struct list_head *head); phys_addr_t acpi_iort_dma_get_max_cpu_address(void); #else -static inline void acpi_iort_init(void) { } static inline u32 iort_msi_map_id(struct device *dev, u32 id) { return id; } static inline struct irq_domain *iort_get_device_domain( -- cgit From f6794950f0e5ba37e3bbedda4d6ab0aad7395dd3 Mon Sep 17 00:00:00 2001 From: Youngmin Nam Date: Mon, 24 Apr 2023 10:04:36 +0900 Subject: arm64: set __exception_irq_entry with __irq_entry as a default filter_irq_stacks() is supposed to cut entries which are related irq entries from its call stack. And in_irqentry_text() which is called by filter_irq_stacks() uses __irqentry_text_start/end symbol to find irq entries in callstack. But it doesn't work correctly as without "CONFIG_FUNCTION_GRAPH_TRACER", arm64 kernel doesn't include gic_handle_irq which is entry point of arm64 irq between __irqentry_text_start and __irqentry_text_end as we discussed in below link. https://lore.kernel.org/all/CACT4Y+aReMGLYua2rCLHgFpS9io5cZC04Q8GLs-uNmrn1ezxYQ@mail.gmail.com/#t This problem can makes unintentional deep call stack entries especially in KASAN enabled situation as below. [ 2479.383395]I[0:launcher-loader: 1719] Stack depot reached limit capacity [ 2479.383538]I[0:launcher-loader: 1719] WARNING: CPU: 0 PID: 1719 at lib/stackdepot.c:129 __stack_depot_save+0x464/0x46c [ 2479.385693]I[0:launcher-loader: 1719] pstate: 624000c5 (nZCv daIF +PAN -UAO +TCO -DIT -SSBS BTYPE=--) [ 2479.385724]I[0:launcher-loader: 1719] pc : __stack_depot_save+0x464/0x46c [ 2479.385751]I[0:launcher-loader: 1719] lr : __stack_depot_save+0x460/0x46c [ 2479.385774]I[0:launcher-loader: 1719] sp : ffffffc0080073c0 [ 2479.385793]I[0:launcher-loader: 1719] x29: ffffffc0080073e0 x28: ffffffd00b78a000 x27: 0000000000000000 [ 2479.385839]I[0:launcher-loader: 1719] x26: 000000000004d1dd x25: ffffff891474f000 x24: 00000000ca64d1dd [ 2479.385882]I[0:launcher-loader: 1719] x23: 0000000000000200 x22: 0000000000000220 x21: 0000000000000040 [ 2479.385925]I[0:launcher-loader: 1719] x20: ffffffc008007440 x19: 0000000000000000 x18: 0000000000000000 [ 2479.385969]I[0:launcher-loader: 1719] x17: 2065726568207475 x16: 000000000000005e x15: 2d2d2d2d2d2d2d20 [ 2479.386013]I[0:launcher-loader: 1719] x14: 5d39313731203a72 x13: 00000000002f6b30 x12: 00000000002f6af8 [ 2479.386057]I[0:launcher-loader: 1719] x11: 00000000ffffffff x10: ffffffb90aacf000 x9 : e8a74a6c16008800 [ 2479.386101]I[0:launcher-loader: 1719] x8 : e8a74a6c16008800 x7 : 00000000002f6b30 x6 : 00000000002f6af8 [ 2479.386145]I[0:launcher-loader: 1719] x5 : ffffffc0080070c8 x4 : ffffffd00b192380 x3 : ffffffd0092b313c [ 2479.386189]I[0:launcher-loader: 1719] x2 : 0000000000000001 x1 : 0000000000000004 x0 : 0000000000000022 [ 2479.386231]I[0:launcher-loader: 1719] Call trace: [ 2479.386248]I[0:launcher-loader: 1719] __stack_depot_save+0x464/0x46c [ 2479.386273]I[0:launcher-loader: 1719] kasan_save_stack+0x58/0x70 [ 2479.386303]I[0:launcher-loader: 1719] save_stack_info+0x34/0x138 [ 2479.386331]I[0:launcher-loader: 1719] kasan_save_free_info+0x18/0x24 [ 2479.386358]I[0:launcher-loader: 1719] ____kasan_slab_free+0x16c/0x170 [ 2479.386385]I[0:launcher-loader: 1719] __kasan_slab_free+0x10/0x20 [ 2479.386410]I[0:launcher-loader: 1719] kmem_cache_free+0x238/0x53c [ 2479.386435]I[0:launcher-loader: 1719] mempool_free_slab+0x1c/0x28 [ 2479.386460]I[0:launcher-loader: 1719] mempool_free+0x7c/0x1a0 [ 2479.386484]I[0:launcher-loader: 1719] bvec_free+0x34/0x80 [ 2479.386514]I[0:launcher-loader: 1719] bio_free+0x60/0x98 [ 2479.386540]I[0:launcher-loader: 1719] bio_put+0x50/0x21c [ 2479.386567]I[0:launcher-loader: 1719] f2fs_write_end_io+0x4ac/0x4d0 [ 2479.386594]I[0:launcher-loader: 1719] bio_endio+0x2dc/0x300 [ 2479.386622]I[0:launcher-loader: 1719] __dm_io_complete+0x324/0x37c [ 2479.386650]I[0:launcher-loader: 1719] dm_io_dec_pending+0x60/0xa4 [ 2479.386676]I[0:launcher-loader: 1719] clone_endio+0xf8/0x2f0 [ 2479.386700]I[0:launcher-loader: 1719] bio_endio+0x2dc/0x300 [ 2479.386727]I[0:launcher-loader: 1719] blk_update_request+0x258/0x63c [ 2479.386754]I[0:launcher-loader: 1719] scsi_end_request+0x50/0x304 [ 2479.386782]I[0:launcher-loader: 1719] scsi_io_completion+0x88/0x160 [ 2479.386808]I[0:launcher-loader: 1719] scsi_finish_command+0x17c/0x194 [ 2479.386833]I[0:launcher-loader: 1719] scsi_complete+0xcc/0x158 [ 2479.386859]I[0:launcher-loader: 1719] blk_mq_complete_request+0x4c/0x5c [ 2479.386885]I[0:launcher-loader: 1719] scsi_done_internal+0xf4/0x1e0 [ 2479.386910]I[0:launcher-loader: 1719] scsi_done+0x14/0x20 [ 2479.386935]I[0:launcher-loader: 1719] ufshcd_compl_one_cqe+0x578/0x71c [ 2479.386963]I[0:launcher-loader: 1719] ufshcd_mcq_poll_cqe_nolock+0xc8/0x150 [ 2479.386991]I[0:launcher-loader: 1719] ufshcd_intr+0x868/0xc0c [ 2479.387017]I[0:launcher-loader: 1719] __handle_irq_event_percpu+0xd0/0x348 [ 2479.387044]I[0:launcher-loader: 1719] handle_irq_event_percpu+0x24/0x74 [ 2479.387068]I[0:launcher-loader: 1719] handle_irq_event+0x74/0xe0 [ 2479.387091]I[0:launcher-loader: 1719] handle_fasteoi_irq+0x174/0x240 [ 2479.387118]I[0:launcher-loader: 1719] handle_irq_desc+0x7c/0x2c0 [ 2479.387147]I[0:launcher-loader: 1719] generic_handle_domain_irq+0x1c/0x28 [ 2479.387174]I[0:launcher-loader: 1719] gic_handle_irq+0x64/0x158 [ 2479.387204]I[0:launcher-loader: 1719] call_on_irq_stack+0x2c/0x54 [ 2479.387231]I[0:launcher-loader: 1719] do_interrupt_handler+0x70/0xa0 [ 2479.387258]I[0:launcher-loader: 1719] el1_interrupt+0x34/0x68 [ 2479.387283]I[0:launcher-loader: 1719] el1h_64_irq_handler+0x18/0x24 [ 2479.387308]I[0:launcher-loader: 1719] el1h_64_irq+0x68/0x6c [ 2479.387332]I[0:launcher-loader: 1719] blk_attempt_bio_merge+0x8/0x170 [ 2479.387356]I[0:launcher-loader: 1719] blk_mq_attempt_bio_merge+0x78/0x98 [ 2479.387383]I[0:launcher-loader: 1719] blk_mq_submit_bio+0x324/0xa40 [ 2479.387409]I[0:launcher-loader: 1719] __submit_bio+0x104/0x138 [ 2479.387436]I[0:launcher-loader: 1719] submit_bio_noacct_nocheck+0x1d0/0x4a0 [ 2479.387462]I[0:launcher-loader: 1719] submit_bio_noacct+0x618/0x804 [ 2479.387487]I[0:launcher-loader: 1719] submit_bio+0x164/0x180 [ 2479.387511]I[0:launcher-loader: 1719] f2fs_submit_read_bio+0xe4/0x1c4 [ 2479.387537]I[0:launcher-loader: 1719] f2fs_mpage_readpages+0x888/0xa4c [ 2479.387563]I[0:launcher-loader: 1719] f2fs_readahead+0xd4/0x19c [ 2479.387587]I[0:launcher-loader: 1719] read_pages+0xb0/0x4ac [ 2479.387614]I[0:launcher-loader: 1719] page_cache_ra_unbounded+0x238/0x288 [ 2479.387642]I[0:launcher-loader: 1719] do_page_cache_ra+0x60/0x6c [ 2479.387669]I[0:launcher-loader: 1719] page_cache_ra_order+0x318/0x364 [ 2479.387695]I[0:launcher-loader: 1719] ondemand_readahead+0x30c/0x3d8 [ 2479.387722]I[0:launcher-loader: 1719] page_cache_sync_ra+0xb4/0xc8 [ 2479.387749]I[0:launcher-loader: 1719] filemap_read+0x268/0xd24 [ 2479.387777]I[0:launcher-loader: 1719] f2fs_file_read_iter+0x1a0/0x62c [ 2479.387806]I[0:launcher-loader: 1719] vfs_read+0x258/0x34c [ 2479.387831]I[0:launcher-loader: 1719] ksys_pread64+0x8c/0xd0 [ 2479.387857]I[0:launcher-loader: 1719] __arm64_sys_pread64+0x48/0x54 [ 2479.387881]I[0:launcher-loader: 1719] invoke_syscall+0x58/0x158 [ 2479.387909]I[0:launcher-loader: 1719] el0_svc_common+0xf0/0x134 [ 2479.387935]I[0:launcher-loader: 1719] do_el0_svc+0x44/0x114 [ 2479.387961]I[0:launcher-loader: 1719] el0_svc+0x2c/0x80 [ 2479.387985]I[0:launcher-loader: 1719] el0t_64_sync_handler+0x48/0x114 [ 2479.388010]I[0:launcher-loader: 1719] el0t_64_sync+0x190/0x194 [ 2479.388038]I[0:launcher-loader: 1719] Kernel panic - not syncing: kernel: panic_on_warn set ... So let's set __exception_irq_entry with __irq_entry as a default. Applying this patch, we can see gic_hande_irq is included in Systemp.map as below. * Before ffffffc008010000 T __do_softirq ffffffc008010000 T __irqentry_text_end ffffffc008010000 T __irqentry_text_start ffffffc008010000 T __softirqentry_text_start ffffffc008010000 T _stext ffffffc00801066c T __softirqentry_text_end ffffffc008010670 T __entry_text_start * After ffffffc008010000 T __irqentry_text_start ffffffc008010000 T _stext ffffffc008010000 t gic_handle_irq ffffffc00801013c t gic_handle_irq ffffffc008010294 T __irqentry_text_end ffffffc008010298 T __do_softirq ffffffc008010298 T __softirqentry_text_start ffffffc008010904 T __softirqentry_text_end ffffffc008010908 T __entry_text_start Signed-off-by: Youngmin Nam Signed-off-by: SEO HOYOUNG Reviewed-by: Mark Rutland Link: https://lore.kernel.org/r/20230424010436.779733-1-youngmin.nam@samsung.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/exception.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index e73af709cb7a..88d8dfeed0db 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -8,16 +8,11 @@ #define __ASM_EXCEPTION_H #include -#include #include #include -#ifdef CONFIG_FUNCTION_GRAPH_TRACER #define __exception_irq_entry __irq_entry -#else -#define __exception_irq_entry __kprobes -#endif static inline unsigned long disr_to_esr(u64 disr) { -- cgit From 6c4dcaddbd36ffc0f29dd874bbddbab30d8edca8 Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 15 May 2023 17:54:16 +0800 Subject: arm64: kdump: simplify the reservation behaviour of crashkernel=,high On arm64, reservation for 'crashkernel=xM,high' is taken by searching for suitable memory region top down. If the 'xM' of crashkernel high memory is reserved from high memory successfully, it will try to reserve crashkernel low memory later accoringly. Otherwise, it will try to search low memory area for the 'xM' suitable region. Please see the details in Documentation/admin-guide/kernel-parameters.txt. While we observed an unexpected case where a reserved region crosses the high and low meomry boundary. E.g on a system with 4G as low memory end, user added the kernel parameters like: 'crashkernel=512M,high', it could finally have [4G-126M, 4G+386M], [1G, 1G+128M] regions in running kernel. The crashkernel high region crossing low and high memory boudary will bring issues: 1) For crashkernel=x,high, if getting crashkernel high region across low and high memory boundary, then user will see two memory regions in low memory, and one memory region in high memory. The two crashkernel low memory regions are confusing as shown in above example. 2) If people explicityly specify "crashkernel=x,high crashkernel=y,low" and y <= 128M, when crashkernel high region crosses low and high memory boundary and the part of crashkernel high reservation below boundary is bigger than y, the expected crahskernel low reservation will be skipped. But the expected crashkernel high reservation is shrank and could not satisfy user space requirement. 3) The crossing boundary behaviour of crahskernel high reservation is different than x86 arch. On x86_64, the low memory end is 4G fixedly, and the memory near 4G is reserved by system, e.g for mapping firmware, pci mapping, so the crashkernel reservation crossing boundary never happens. From distros point of view, this brings inconsistency and confusion. Users need to dig into x86 and arm64 system details to find out why. For kernel itself, the impact of issue 3) could be slight. While issue 1) and 2) cause actual impact because it brings obscure semantics and behaviour to crashkernel=,high reservation. Here, for crashkernel=xM,high, search the high memory for the suitable region only in high memory. If failed, try reserving the suitable region only in low memory. Like this, the crashkernel high region will only exist in high memory, and crashkernel low region only exists in low memory. The reservation behaviour for crashkernel=,high is clearer and simpler. Note: RPi4 has different zone ranges than normal memory. Its DMA zone is 0~1G, and DMA32 zone is 1G~4G if CONFIG_ZONE_DMA|DMA32 are enabled by default. The low memory end is 1G in order to validate all devices, high memory starts at 1G memory. However, for being consistent with normal arm64 system, its low memory end is still 1G, while reserving crashkernel high memory from 4G if crashkernel=size,high specified. This will remove confusion. With above change applied, summary of arm64 crashkernel reservation range: 1) RPi4(zone DMA:0~1G; DMA32:1G~4G): crashkernel=size 0~1G: low memory | 1G~top: high memory crashkernel=size,high 0~1G: low memory | 4G~top: high memory 2) Other normal system: crashkernel=size crashkernel=size,high 0~4G: low memory | 4G~top: high memory 3) Systems w/o zone DMA|DMA32 crashkernel=size crashkernel=size,high 0~top: low memory Signed-off-by: Baoquan He Reviewed-by: Catalin Marinas Reviewed-by: Zhen Lei Link: https://lore.kernel.org/r/ZGIBSEoZ7VRVvP8H@MiWiFi-R3L-srv Signed-off-by: Catalin Marinas --- arch/arm64/mm/init.c | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 66e70ca47680..c28c2c8483cc 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -69,6 +69,7 @@ phys_addr_t __ro_after_init arm64_dma_phys_limit; #define CRASH_ADDR_LOW_MAX arm64_dma_phys_limit #define CRASH_ADDR_HIGH_MAX (PHYS_MASK + 1) +#define CRASH_HIGH_SEARCH_BASE SZ_4G #define DEFAULT_CRASH_KERNEL_LOW_SIZE (128UL << 20) @@ -101,12 +102,13 @@ static int __init reserve_crashkernel_low(unsigned long long low_size) */ static void __init reserve_crashkernel(void) { - unsigned long long crash_base, crash_size; - unsigned long long crash_low_size = 0; + unsigned long long crash_low_size = 0, search_base = 0; unsigned long long crash_max = CRASH_ADDR_LOW_MAX; + unsigned long long crash_base, crash_size; char *cmdline = boot_command_line; - int ret; bool fixed_base = false; + bool high = false; + int ret; if (!IS_ENABLED(CONFIG_KEXEC_CORE)) return; @@ -129,7 +131,9 @@ static void __init reserve_crashkernel(void) else if (ret) return; + search_base = CRASH_HIGH_SEARCH_BASE; crash_max = CRASH_ADDR_HIGH_MAX; + high = true; } else if (ret || !crash_size) { /* The specified value is invalid */ return; @@ -140,31 +144,51 @@ static void __init reserve_crashkernel(void) /* User specifies base address explicitly. */ if (crash_base) { fixed_base = true; + search_base = crash_base; crash_max = crash_base + crash_size; } retry: crash_base = memblock_phys_alloc_range(crash_size, CRASH_ALIGN, - crash_base, crash_max); + search_base, crash_max); if (!crash_base) { /* - * If the first attempt was for low memory, fall back to - * high memory, the minimum required low memory will be - * reserved later. + * For crashkernel=size[KMG]@offset[KMG], print out failure + * message if can't reserve the specified region. */ - if (!fixed_base && (crash_max == CRASH_ADDR_LOW_MAX)) { + if (fixed_base) { + pr_warn("crashkernel reservation failed - memory is in use.\n"); + return; + } + + /* + * For crashkernel=size[KMG], if the first attempt was for + * low memory, fall back to high memory, the minimum required + * low memory will be reserved later. + */ + if (!high && crash_max == CRASH_ADDR_LOW_MAX) { crash_max = CRASH_ADDR_HIGH_MAX; + search_base = CRASH_ADDR_LOW_MAX; crash_low_size = DEFAULT_CRASH_KERNEL_LOW_SIZE; goto retry; } + /* + * For crashkernel=size[KMG],high, if the first attempt was + * for high memory, fall back to low memory. + */ + if (high && crash_max == CRASH_ADDR_HIGH_MAX) { + crash_max = CRASH_ADDR_LOW_MAX; + search_base = 0; + goto retry; + } pr_warn("cannot allocate crashkernel (size:0x%llx)\n", crash_size); return; } - if ((crash_base > CRASH_ADDR_LOW_MAX - crash_low_size) && - crash_low_size && reserve_crashkernel_low(crash_low_size)) { + if ((crash_base >= CRASH_ADDR_LOW_MAX) && crash_low_size && + reserve_crashkernel_low(crash_low_size)) { memblock_phys_free(crash_base, crash_size); return; } -- cgit From 03dc0e05407f394d4f8b3da8210013fef91e74ff Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 15 May 2023 14:02:59 +0800 Subject: Documentation: add kdump.rst to present crashkernel reservation on arm64 People complained the crashkernel reservation code flow is hard to follow, so add this document to explain the background, concepts and implementation of crashkernel reservation on arm64. Hope this can help people to understand it more easily. Signed-off-by: Baoquan He Reviewed-by: Zhen Lei Link: https://lore.kernel.org/r/20230515060259.830662-3-bhe@redhat.com [catalin.marinas@arm.com: rewrote some sentences, removed others] Signed-off-by: Catalin Marinas --- Documentation/arm64/kdump.rst | 92 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 Documentation/arm64/kdump.rst diff --git a/Documentation/arm64/kdump.rst b/Documentation/arm64/kdump.rst new file mode 100644 index 000000000000..56a89f45df28 --- /dev/null +++ b/Documentation/arm64/kdump.rst @@ -0,0 +1,92 @@ +======================================= +crashkernel memory reservation on arm64 +======================================= + +Author: Baoquan He + +Kdump mechanism is used to capture a corrupted kernel vmcore so that +it can be subsequently analyzed. In order to do this, a preliminarily +reserved memory is needed to pre-load the kdump kernel and boot such +kernel if corruption happens. + +That reserved memory for kdump is adapted to be able to minimally +accommodate the kdump kernel and the user space programs needed for the +vmcore collection. + +Kernel parameter +================ + +Through the kernel parameters below, memory can be reserved accordingly +during the early stage of the first kernel booting so that a continuous +large chunk of memomy can be found. The low memory reservation needs to +be considered if the crashkernel is reserved from the high memory area. + +- crashkernel=size@offset +- crashkernel=size +- crashkernel=size,high crashkernel=size,low + +Low memory and high memory +========================== + +For kdump reservations, low memory is the memory area under a specific +limit, usually decided by the accessible address bits of the DMA-capable +devices needed by the kdump kernel to run. Those devices not related to +vmcore dumping can be ignored. On arm64, the low memory upper bound is +not fixed: it is 1G on the RPi4 platform but 4G on most other systems. +On special kernels built with CONFIG_ZONE_(DMA|DMA32) disabled, the +whole system RAM is low memory. Outside of the low memory described +above, the rest of system RAM is considered high memory. + +Implementation +============== + +1) crashkernel=size@offset +-------------------------- + +The crashkernel memory must be reserved at the user-specified region or +fail if already occupied. + + +2) crashkernel=size +------------------- + +The crashkernel memory region will be reserved in any available position +according to the search order: + +Firstly, the kernel searches the low memory area for an available region +with the specified size. + +If searching for low memory fails, the kernel falls back to searching +the high memory area for an available region of the specified size. If +the reservation in high memory succeeds, a default size reservation in +the low memory will be done. Currently the default size is 128M, +sufficient for the low memory needs of the kdump kernel. + +Note: crashkernel=size is the recommended option for crashkernel kernel +reservations. The user would not need to know the system memory layout +for a specific platform. + +3) crashkernel=size,high crashkernel=size,low +--------------------------------------------- + +crashkernel=size,(high|low) are an important supplement to +crashkernel=size. They allows the user to specify how much memory needs +to be allocated from the high memory and low memory respectively. On +many systems the low memory is precious and crashkernel reservations +from this area should be kept to a minimum. + +To reserve memory for crashkernel=size,high, searching is first +attempted from the high memory region. If the reservation succeeds, the +low memory reservation will be done subsequently. + +If reservation from the high memory failed, the kernel falls back to +searching the low memory with the specified size in crashkernel=,high. +If it succeeds, no further reservation for low memory is needed. + +Notes: + +- If crashkernel=,low is not specified, the default low memory + reservation will be done automatically. + +- if crashkernel=0,low is specified, it means that the low memory + reservation is omitted intentionally. -- cgit From 389ce21b622b0dba4e9134d6236ce0bf1635edcb Mon Sep 17 00:00:00 2001 From: Baoquan He Date: Mon, 12 Jun 2023 07:03:58 +0800 Subject: arm64: add kdump.rst into index.rst Document kdump.rst was added into Documentation/arm64/, but not listed in Documentation/arm64/index.rst. That triggers below warning when executing "make htmldoc": >> Documentation/arm64/kdump.rst: WARNING: document isn't included in any toctree Fix it now. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202306110549.ynH2Juok-lkp@intel.com/ Signed-off-by: Baoquan He Fixes: 03dc0e05407f ("Documentation: add kdump.rst to present crashkernel reservation on arm64") Reviewed-by: Bagas Sanjaya Link: https://lore.kernel.org/r/20230611230358.13635-1-bhe@redhat.com Signed-off-by: Catalin Marinas --- Documentation/arm64/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/arm64/index.rst b/Documentation/arm64/index.rst index ae21f8118830..e58391557f00 100644 --- a/Documentation/arm64/index.rst +++ b/Documentation/arm64/index.rst @@ -15,6 +15,7 @@ ARM64 Architecture cpu-feature-registers elf_hwcaps hugetlbpage + kdump legacy_instructions memory memory-tagging-extension -- cgit From 92b1efcd9d9d984af1e60ae4f575eb2c5bfea303 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:36 +0530 Subject: arm64/sysreg: Rename TRBLIMITR_EL1 fields per auto-gen tools format This renames TRBLIMITR_EL1 register fields per auto-gen tools format without causing any functional change in the TRBE driver. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: kvmarm@lists.linux.dev Cc: coresight@lists.linaro.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-2-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 16 ++++++++-------- arch/arm64/kvm/hyp/nvhe/debug-sr.c | 2 +- drivers/hwtracing/coresight/coresight-trbe.c | 21 +++++++++++---------- drivers/hwtracing/coresight/coresight-trbe.h | 7 ++++--- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 23a17da500a4..d930400ea94f 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -238,14 +238,14 @@ #define SYS_TRBTRG_EL1 sys_reg(3, 0, 9, 11, 6) #define SYS_TRBIDR_EL1 sys_reg(3, 0, 9, 11, 7) -#define TRBLIMITR_LIMIT_MASK GENMASK_ULL(51, 0) -#define TRBLIMITR_LIMIT_SHIFT 12 -#define TRBLIMITR_NVM BIT(5) -#define TRBLIMITR_TRIG_MODE_MASK GENMASK(1, 0) -#define TRBLIMITR_TRIG_MODE_SHIFT 3 -#define TRBLIMITR_FILL_MODE_MASK GENMASK(1, 0) -#define TRBLIMITR_FILL_MODE_SHIFT 1 -#define TRBLIMITR_ENABLE BIT(0) +#define TRBLIMITR_EL1_LIMIT_MASK GENMASK_ULL(63, 12) +#define TRBLIMITR_EL1_LIMIT_SHIFT 12 +#define TRBLIMITR_EL1_nVM BIT(5) +#define TRBLIMITR_EL1_TM_MASK GENMASK(4, 3) +#define TRBLIMITR_EL1_TM_SHIFT 3 +#define TRBLIMITR_EL1_FM_MASK GENMASK(2, 1) +#define TRBLIMITR_EL1_FM_SHIFT 1 +#define TRBLIMITR_EL1_E BIT(0) #define TRBPTR_PTR_MASK GENMASK_ULL(63, 0) #define TRBPTR_PTR_SHIFT 0 #define TRBBASER_BASE_MASK GENMASK_ULL(51, 0) diff --git a/arch/arm64/kvm/hyp/nvhe/debug-sr.c b/arch/arm64/kvm/hyp/nvhe/debug-sr.c index d756b939f296..4558c02eb352 100644 --- a/arch/arm64/kvm/hyp/nvhe/debug-sr.c +++ b/arch/arm64/kvm/hyp/nvhe/debug-sr.c @@ -56,7 +56,7 @@ static void __debug_save_trace(u64 *trfcr_el1) *trfcr_el1 = 0; /* Check if the TRBE is enabled */ - if (!(read_sysreg_s(SYS_TRBLIMITR_EL1) & TRBLIMITR_ENABLE)) + if (!(read_sysreg_s(SYS_TRBLIMITR_EL1) & TRBLIMITR_EL1_E)) return; /* * Prohibit trace generation while we are in guest. diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 1fc4fd79a1c6..1d9d141c62e9 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -218,7 +218,7 @@ static inline void set_trbe_enabled(struct trbe_cpudata *cpudata, u64 trblimitr) * Enable the TRBE without clearing LIMITPTR which * might be required for fetching the buffer limits. */ - trblimitr |= TRBLIMITR_ENABLE; + trblimitr |= TRBLIMITR_EL1_E; write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1); /* Synchronize the TRBE enable event */ @@ -236,7 +236,7 @@ static inline void set_trbe_disabled(struct trbe_cpudata *cpudata) * Disable the TRBE without clearing LIMITPTR which * might be required for fetching the buffer limits. */ - trblimitr &= ~TRBLIMITR_ENABLE; + trblimitr &= ~TRBLIMITR_EL1_E; write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1); if (trbe_needs_drain_after_disable(cpudata)) @@ -596,13 +596,13 @@ static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf) u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1); unsigned long addr = buf->trbe_limit; - WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_LIMIT_SHIFT))); + WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_EL1_LIMIT_SHIFT))); WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE)); - trblimitr &= ~TRBLIMITR_NVM; - trblimitr &= ~(TRBLIMITR_FILL_MODE_MASK << TRBLIMITR_FILL_MODE_SHIFT); - trblimitr &= ~(TRBLIMITR_TRIG_MODE_MASK << TRBLIMITR_TRIG_MODE_SHIFT); - trblimitr &= ~(TRBLIMITR_LIMIT_MASK << TRBLIMITR_LIMIT_SHIFT); + trblimitr &= ~TRBLIMITR_EL1_nVM; + trblimitr &= ~TRBLIMITR_EL1_FM_MASK; + trblimitr &= ~TRBLIMITR_EL1_TM_MASK; + trblimitr &= ~TRBLIMITR_EL1_LIMIT_MASK; /* * Fill trace buffer mode is used here while configuring the @@ -613,14 +613,15 @@ static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf) * trace data in the interrupt handler, before reconfiguring * the TRBE. */ - trblimitr |= (TRBE_FILL_MODE_FILL & TRBLIMITR_FILL_MODE_MASK) << TRBLIMITR_FILL_MODE_SHIFT; + trblimitr |= (TRBLIMITR_EL1_FM_FILL << TRBLIMITR_EL1_FM_SHIFT) & + TRBLIMITR_EL1_FM_MASK; /* * Trigger mode is not used here while configuring the TRBE for * the trace capture. Hence just keep this in the ignore mode. */ - trblimitr |= (TRBE_TRIG_MODE_IGNORE & TRBLIMITR_TRIG_MODE_MASK) << - TRBLIMITR_TRIG_MODE_SHIFT; + trblimitr |= (TRBLIMITR_EL1_TM_IGNR << TRBLIMITR_EL1_TM_SHIFT) & + TRBLIMITR_EL1_TM_MASK; trblimitr |= (addr & PAGE_MASK); set_trbe_enabled(buf->cpudata, trblimitr); } diff --git a/drivers/hwtracing/coresight/coresight-trbe.h b/drivers/hwtracing/coresight/coresight-trbe.h index 98ff1b17ad07..8ea7079d60bb 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.h +++ b/drivers/hwtracing/coresight/coresight-trbe.h @@ -30,7 +30,7 @@ static inline bool is_trbe_enabled(void) { u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1); - return trblimitr & TRBLIMITR_ENABLE; + return trblimitr & TRBLIMITR_EL1_E; } #define TRBE_EC_OTHERS 0 @@ -86,8 +86,9 @@ static inline bool is_trbe_running(u64 trbsr) #define TRBE_TRIG_MODE_STOP 0 #define TRBE_TRIG_MODE_IRQ 1 -#define TRBE_TRIG_MODE_IGNORE 3 +#define TRBLIMITR_EL1_TM_IGNR 3 +#define TRBLIMITR_EL1_FM_FILL 0 #define TRBE_FILL_MODE_FILL 0 #define TRBE_FILL_MODE_WRAP 1 #define TRBE_FILL_MODE_CIRCULAR_BUFFER 3 @@ -121,7 +122,7 @@ static inline void set_trbe_write_pointer(unsigned long addr) static inline unsigned long get_trbe_limit_pointer(void) { u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1); - unsigned long addr = trblimitr & (TRBLIMITR_LIMIT_MASK << TRBLIMITR_LIMIT_SHIFT); + unsigned long addr = trblimitr & TRBLIMITR_EL1_LIMIT_MASK; WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE)); return addr; -- cgit From e01e1737e348acdfc43a620060410559072f29c1 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:37 +0530 Subject: arm64/sysreg: Rename TRBPTR_EL1 fields per auto-gen tools format This renames TRBPTR_EL1 register fields per auto-gen tools format without causing any functional change in the TRBE driver. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: kvmarm@lists.linux.dev Cc: coresight@lists.linaro.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-3-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index d930400ea94f..31005f72353e 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -246,8 +246,8 @@ #define TRBLIMITR_EL1_FM_MASK GENMASK(2, 1) #define TRBLIMITR_EL1_FM_SHIFT 1 #define TRBLIMITR_EL1_E BIT(0) -#define TRBPTR_PTR_MASK GENMASK_ULL(63, 0) -#define TRBPTR_PTR_SHIFT 0 +#define TRBPTR_EL1_PTR_MASK GENMASK_ULL(63, 0) +#define TRBPTR_EL1_PTR_SHIFT 0 #define TRBBASER_BASE_MASK GENMASK_ULL(51, 0) #define TRBBASER_BASE_SHIFT 12 #define TRBSR_EC_MASK GENMASK(5, 0) -- cgit From 90cdde836c43154acb474553095bb7ee741160a5 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:38 +0530 Subject: arm64/sysreg: Rename TRBBASER_EL1 fields per auto-gen tools format This renames TRBBASER_EL1 register fields per auto-gen tools format without causing any functional change in the TRBE driver. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: kvmarm@lists.linux.dev Cc: coresight@lists.linaro.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-4-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 4 ++-- drivers/hwtracing/coresight/coresight-trbe.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 31005f72353e..80780dec31d0 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -248,8 +248,8 @@ #define TRBLIMITR_EL1_E BIT(0) #define TRBPTR_EL1_PTR_MASK GENMASK_ULL(63, 0) #define TRBPTR_EL1_PTR_SHIFT 0 -#define TRBBASER_BASE_MASK GENMASK_ULL(51, 0) -#define TRBBASER_BASE_SHIFT 12 +#define TRBBASER_EL1_BASE_MASK GENMASK_ULL(63, 12) +#define TRBBASER_EL1_BASE_SHIFT 12 #define TRBSR_EC_MASK GENMASK(5, 0) #define TRBSR_EC_SHIFT 26 #define TRBSR_IRQ BIT(22) diff --git a/drivers/hwtracing/coresight/coresight-trbe.h b/drivers/hwtracing/coresight/coresight-trbe.h index 8ea7079d60bb..0b73d9d10aa8 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.h +++ b/drivers/hwtracing/coresight/coresight-trbe.h @@ -131,7 +131,7 @@ static inline unsigned long get_trbe_limit_pointer(void) static inline unsigned long get_trbe_base_pointer(void) { u64 trbbaser = read_sysreg_s(SYS_TRBBASER_EL1); - unsigned long addr = trbbaser & (TRBBASER_BASE_MASK << TRBBASER_BASE_SHIFT); + unsigned long addr = trbbaser & TRBBASER_EL1_BASE_MASK; WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE)); return addr; @@ -140,7 +140,7 @@ static inline unsigned long get_trbe_base_pointer(void) static inline void set_trbe_base_pointer(unsigned long addr) { WARN_ON(is_trbe_enabled()); - WARN_ON(!IS_ALIGNED(addr, (1UL << TRBBASER_BASE_SHIFT))); + WARN_ON(!IS_ALIGNED(addr, (1UL << TRBBASER_EL1_BASE_SHIFT))); WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE)); write_sysreg_s(addr, SYS_TRBBASER_EL1); } -- cgit From 7bb948826610f05b42567ce89156d6513d53d988 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:39 +0530 Subject: arm64/sysreg: Rename TRBSR_EL1 fields per auto-gen tools format This renames TRBSR_EL1 register fields per auto-gen tools format without causing any functional change in the TRBE driver. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: kvmarm@lists.linux.dev Cc: coresight@lists.linaro.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-5-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 26 +++++++++++++------------- drivers/hwtracing/coresight/coresight-trbe.c | 12 ++++++------ drivers/hwtracing/coresight/coresight-trbe.h | 16 ++++++++-------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 80780dec31d0..1df57d672a65 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -250,19 +250,19 @@ #define TRBPTR_EL1_PTR_SHIFT 0 #define TRBBASER_EL1_BASE_MASK GENMASK_ULL(63, 12) #define TRBBASER_EL1_BASE_SHIFT 12 -#define TRBSR_EC_MASK GENMASK(5, 0) -#define TRBSR_EC_SHIFT 26 -#define TRBSR_IRQ BIT(22) -#define TRBSR_TRG BIT(21) -#define TRBSR_WRAP BIT(20) -#define TRBSR_ABORT BIT(18) -#define TRBSR_STOP BIT(17) -#define TRBSR_MSS_MASK GENMASK(15, 0) -#define TRBSR_MSS_SHIFT 0 -#define TRBSR_BSC_MASK GENMASK(5, 0) -#define TRBSR_BSC_SHIFT 0 -#define TRBSR_FSC_MASK GENMASK(5, 0) -#define TRBSR_FSC_SHIFT 0 +#define TRBSR_EL1_EC_MASK GENMASK(31, 26) +#define TRBSR_EL1_EC_SHIFT 26 +#define TRBSR_EL1_IRQ BIT(22) +#define TRBSR_EL1_TRG BIT(21) +#define TRBSR_EL1_WRAP BIT(20) +#define TRBSR_EL1_EA BIT(18) +#define TRBSR_EL1_S BIT(17) +#define TRBSR_EL1_MSS_MASK GENMASK(15, 0) +#define TRBSR_EL1_MSS_SHIFT 0 +#define TRBSR_EL1_BSC_MASK GENMASK(5, 0) +#define TRBSR_EL1_BSC_SHIFT 0 +#define TRBSR_EL1_FSC_MASK GENMASK(5, 0) +#define TRBSR_EL1_FSC_SHIFT 0 #define TRBMAR_SHARE_MASK GENMASK(1, 0) #define TRBMAR_SHARE_SHIFT 8 #define TRBMAR_OUTER_MASK GENMASK(3, 0) diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 1d9d141c62e9..1bab91ce8e95 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -582,12 +582,12 @@ static void clr_trbe_status(void) u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1); WARN_ON(is_trbe_enabled()); - trbsr &= ~TRBSR_IRQ; - trbsr &= ~TRBSR_TRG; - trbsr &= ~TRBSR_WRAP; - trbsr &= ~(TRBSR_EC_MASK << TRBSR_EC_SHIFT); - trbsr &= ~(TRBSR_BSC_MASK << TRBSR_BSC_SHIFT); - trbsr &= ~TRBSR_STOP; + trbsr &= ~TRBSR_EL1_IRQ; + trbsr &= ~TRBSR_EL1_TRG; + trbsr &= ~TRBSR_EL1_WRAP; + trbsr &= ~TRBSR_EL1_EC_MASK; + trbsr &= ~TRBSR_EL1_BSC_MASK; + trbsr &= ~TRBSR_EL1_S; write_sysreg_s(trbsr, SYS_TRBSR_EL1); } diff --git a/drivers/hwtracing/coresight/coresight-trbe.h b/drivers/hwtracing/coresight/coresight-trbe.h index 0b73d9d10aa8..3743d9085355 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.h +++ b/drivers/hwtracing/coresight/coresight-trbe.h @@ -39,7 +39,7 @@ static inline bool is_trbe_enabled(void) static inline int get_trbe_ec(u64 trbsr) { - return (trbsr >> TRBSR_EC_SHIFT) & TRBSR_EC_MASK; + return (trbsr & TRBSR_EL1_EC_MASK) >> TRBSR_EL1_EC_SHIFT; } #define TRBE_BSC_NOT_STOPPED 0 @@ -48,40 +48,40 @@ static inline int get_trbe_ec(u64 trbsr) static inline int get_trbe_bsc(u64 trbsr) { - return (trbsr >> TRBSR_BSC_SHIFT) & TRBSR_BSC_MASK; + return (trbsr & TRBSR_EL1_BSC_MASK) >> TRBSR_EL1_BSC_SHIFT; } static inline void clr_trbe_irq(void) { u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1); - trbsr &= ~TRBSR_IRQ; + trbsr &= ~TRBSR_EL1_IRQ; write_sysreg_s(trbsr, SYS_TRBSR_EL1); } static inline bool is_trbe_irq(u64 trbsr) { - return trbsr & TRBSR_IRQ; + return trbsr & TRBSR_EL1_IRQ; } static inline bool is_trbe_trg(u64 trbsr) { - return trbsr & TRBSR_TRG; + return trbsr & TRBSR_EL1_TRG; } static inline bool is_trbe_wrap(u64 trbsr) { - return trbsr & TRBSR_WRAP; + return trbsr & TRBSR_EL1_WRAP; } static inline bool is_trbe_abort(u64 trbsr) { - return trbsr & TRBSR_ABORT; + return trbsr & TRBSR_EL1_EA; } static inline bool is_trbe_running(u64 trbsr) { - return !(trbsr & TRBSR_STOP); + return !(trbsr & TRBSR_EL1_S); } #define TRBE_TRIG_MODE_STOP 0 -- cgit From b7c3a6eb4d2b6abf9aa2b08d70e5fc9008797a86 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:40 +0530 Subject: arm64/sysreg: Rename TRBMAR_EL1 fields per auto-gen tools format This renames TRBMAR_EL1 register fields per auto-gen tools format without causing any functional change in the TRBE driver. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: kvmarm@lists.linux.dev Cc: coresight@lists.linaro.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-6-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 1df57d672a65..ef8c9d81b6ad 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -263,12 +263,10 @@ #define TRBSR_EL1_BSC_SHIFT 0 #define TRBSR_EL1_FSC_MASK GENMASK(5, 0) #define TRBSR_EL1_FSC_SHIFT 0 -#define TRBMAR_SHARE_MASK GENMASK(1, 0) -#define TRBMAR_SHARE_SHIFT 8 -#define TRBMAR_OUTER_MASK GENMASK(3, 0) -#define TRBMAR_OUTER_SHIFT 4 -#define TRBMAR_INNER_MASK GENMASK(3, 0) -#define TRBMAR_INNER_SHIFT 0 +#define TRBMAR_EL1_SH_MASK GENMASK(9, 8) +#define TRBMAR_EL1_SH_SHIFT 8 +#define TRBMAR_EL1_Attr_MASK GENMASK(7, 0) +#define TRBMAR_EL1_Attr_SHIFT 0 #define TRBTRG_TRG_MASK GENMASK(31, 0) #define TRBTRG_TRG_SHIFT 0 #define TRBIDR_FLAG BIT(5) -- cgit From dae169fd63f323b4dea1a01beb46c558e10315ac Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:41 +0530 Subject: arm64/sysreg: Rename TRBTRG_EL1 fields per auto-gen tools format This renames TRBTRG_EL1 register fields per auto-gen tools format without causing any functional change in the TRBE driver. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: kvmarm@lists.linux.dev Cc: coresight@lists.linaro.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-7-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index ef8c9d81b6ad..0ba0f268ac79 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -267,8 +267,8 @@ #define TRBMAR_EL1_SH_SHIFT 8 #define TRBMAR_EL1_Attr_MASK GENMASK(7, 0) #define TRBMAR_EL1_Attr_SHIFT 0 -#define TRBTRG_TRG_MASK GENMASK(31, 0) -#define TRBTRG_TRG_SHIFT 0 +#define TRBTRG_EL1_TRG_MASK GENMASK(31, 0) +#define TRBTRG_EL1_TRG_SHIFT 0 #define TRBIDR_FLAG BIT(5) #define TRBIDR_PROG BIT(4) #define TRBIDR_ALIGN_MASK GENMASK(3, 0) -- cgit From f170aa51e6c53f49e90805d1fffb55fd199f82b3 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:42 +0530 Subject: arm64/sysreg: Rename TRBIDR_EL1 fields per auto-gen tools format This renames TRBIDR_EL1 register fields per auto-gen tools format without causing any functional change in the TRBE driver. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: kvmarm@lists.linux.dev Cc: coresight@lists.linaro.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-8-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/el2_setup.h | 2 +- arch/arm64/include/asm/sysreg.h | 8 ++++---- arch/arm64/kvm/debug.c | 2 +- drivers/hwtracing/coresight/coresight-trbe.h | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 037724b19c5c..63ea1ef6c99e 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -69,7 +69,7 @@ cbz x0, .Lskip_trace_\@ // Skip if TraceBuffer is not present mrs_s x0, SYS_TRBIDR_EL1 - and x0, x0, TRBIDR_PROG + and x0, x0, TRBIDR_EL1_P cbnz x0, .Lskip_trace_\@ // If TRBE is available at EL2 mov x0, #(MDCR_EL2_E2TB_MASK << MDCR_EL2_E2TB_SHIFT) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 0ba0f268ac79..d699747a0363 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -269,10 +269,10 @@ #define TRBMAR_EL1_Attr_SHIFT 0 #define TRBTRG_EL1_TRG_MASK GENMASK(31, 0) #define TRBTRG_EL1_TRG_SHIFT 0 -#define TRBIDR_FLAG BIT(5) -#define TRBIDR_PROG BIT(4) -#define TRBIDR_ALIGN_MASK GENMASK(3, 0) -#define TRBIDR_ALIGN_SHIFT 0 +#define TRBIDR_EL1_F BIT(5) +#define TRBIDR_EL1_P BIT(4) +#define TRBIDR_EL1_Align_MASK GENMASK(3, 0) +#define TRBIDR_EL1_Align_SHIFT 0 #define SYS_PMINTENSET_EL1 sys_reg(3, 0, 9, 14, 1) #define SYS_PMINTENCLR_EL1 sys_reg(3, 0, 9, 14, 2) diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c index 55f80fb93925..8725291cb00a 100644 --- a/arch/arm64/kvm/debug.c +++ b/arch/arm64/kvm/debug.c @@ -333,7 +333,7 @@ void kvm_arch_vcpu_load_debug_state_flags(struct kvm_vcpu *vcpu) /* Check if we have TRBE implemented and available at the host */ if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_TraceBuffer_SHIFT) && - !(read_sysreg_s(SYS_TRBIDR_EL1) & TRBIDR_PROG)) + !(read_sysreg_s(SYS_TRBIDR_EL1) & TRBIDR_EL1_P)) vcpu_set_flag(vcpu, DEBUG_STATE_SAVE_TRBE); } diff --git a/drivers/hwtracing/coresight/coresight-trbe.h b/drivers/hwtracing/coresight/coresight-trbe.h index 3743d9085355..d661b062293f 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.h +++ b/drivers/hwtracing/coresight/coresight-trbe.h @@ -95,17 +95,17 @@ static inline bool is_trbe_running(u64 trbsr) static inline bool get_trbe_flag_update(u64 trbidr) { - return trbidr & TRBIDR_FLAG; + return trbidr & TRBIDR_EL1_F; } static inline bool is_trbe_programmable(u64 trbidr) { - return !(trbidr & TRBIDR_PROG); + return !(trbidr & TRBIDR_EL1_P); } static inline int get_trbe_address_align(u64 trbidr) { - return (trbidr >> TRBIDR_ALIGN_SHIFT) & TRBIDR_ALIGN_MASK; + return (trbidr & TRBIDR_EL1_Align_MASK) >> TRBIDR_EL1_Align_SHIFT; } static inline unsigned long get_trbe_write_pointer(void) -- cgit From eee64165a54e8e045fffa41950b60af51856266c Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:43 +0530 Subject: arm64/sysreg: Convert TRBLIMITR_EL1 register to automatic generation This converts TRBLIMITR_EL1 register to automatic generation without causing any functional change. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Reviewed-by: Mark Brown Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-9-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 12 ------------ arch/arm64/tools/sysreg | 18 ++++++++++++++++++ drivers/hwtracing/coresight/coresight-trbe.h | 9 --------- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index d699747a0363..f97d08d8d8da 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -227,10 +227,6 @@ /*** End of Statistical Profiling Extension ***/ -/* - * TRBE Registers - */ -#define SYS_TRBLIMITR_EL1 sys_reg(3, 0, 9, 11, 0) #define SYS_TRBPTR_EL1 sys_reg(3, 0, 9, 11, 1) #define SYS_TRBBASER_EL1 sys_reg(3, 0, 9, 11, 2) #define SYS_TRBSR_EL1 sys_reg(3, 0, 9, 11, 3) @@ -238,14 +234,6 @@ #define SYS_TRBTRG_EL1 sys_reg(3, 0, 9, 11, 6) #define SYS_TRBIDR_EL1 sys_reg(3, 0, 9, 11, 7) -#define TRBLIMITR_EL1_LIMIT_MASK GENMASK_ULL(63, 12) -#define TRBLIMITR_EL1_LIMIT_SHIFT 12 -#define TRBLIMITR_EL1_nVM BIT(5) -#define TRBLIMITR_EL1_TM_MASK GENMASK(4, 3) -#define TRBLIMITR_EL1_TM_SHIFT 3 -#define TRBLIMITR_EL1_FM_MASK GENMASK(2, 1) -#define TRBLIMITR_EL1_FM_SHIFT 1 -#define TRBLIMITR_EL1_E BIT(0) #define TRBPTR_EL1_PTR_MASK GENMASK_ULL(63, 0) #define TRBPTR_EL1_PTR_SHIFT 0 #define TRBBASER_EL1_BASE_MASK GENMASK_ULL(63, 12) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 41462785020b..fe2490713754 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2255,3 +2255,21 @@ Sysreg ICC_NMIAR1_EL1 3 0 12 9 5 Res0 63:24 Field 23:0 INTID EndSysreg + +Sysreg TRBLIMITR_EL1 3 0 9 11 0 +Field 63:12 LIMIT +Res0 11:7 +Field 6 XE +Field 5 nVM +Enum 4:3 TM + 0b00 STOP + 0b01 IRQ + 0b11 IGNR +EndEnum +Enum 2:1 FM + 0b00 FILL + 0b01 WRAP + 0b11 CBUF +EndEnum +Field 0 E +EndSysreg diff --git a/drivers/hwtracing/coresight/coresight-trbe.h b/drivers/hwtracing/coresight/coresight-trbe.h index d661b062293f..77cbb5c63878 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.h +++ b/drivers/hwtracing/coresight/coresight-trbe.h @@ -84,15 +84,6 @@ static inline bool is_trbe_running(u64 trbsr) return !(trbsr & TRBSR_EL1_S); } -#define TRBE_TRIG_MODE_STOP 0 -#define TRBE_TRIG_MODE_IRQ 1 -#define TRBLIMITR_EL1_TM_IGNR 3 - -#define TRBLIMITR_EL1_FM_FILL 0 -#define TRBE_FILL_MODE_FILL 0 -#define TRBE_FILL_MODE_WRAP 1 -#define TRBE_FILL_MODE_CIRCULAR_BUFFER 3 - static inline bool get_trbe_flag_update(u64 trbidr) { return trbidr & TRBIDR_EL1_F; -- cgit From 6669697733ca50d9be9e14cdfd2318bc37d84d97 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:44 +0530 Subject: arm64/sysreg: Convert TRBPTR_EL1 register to automatic generation This converts TRBPTR_EL1 register to automatic generation without causing any functional change. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Reviewed-by: Mark Brown Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-10-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 3 --- arch/arm64/tools/sysreg | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index f97d08d8d8da..6b3204fbad22 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -227,15 +227,12 @@ /*** End of Statistical Profiling Extension ***/ -#define SYS_TRBPTR_EL1 sys_reg(3, 0, 9, 11, 1) #define SYS_TRBBASER_EL1 sys_reg(3, 0, 9, 11, 2) #define SYS_TRBSR_EL1 sys_reg(3, 0, 9, 11, 3) #define SYS_TRBMAR_EL1 sys_reg(3, 0, 9, 11, 4) #define SYS_TRBTRG_EL1 sys_reg(3, 0, 9, 11, 6) #define SYS_TRBIDR_EL1 sys_reg(3, 0, 9, 11, 7) -#define TRBPTR_EL1_PTR_MASK GENMASK_ULL(63, 0) -#define TRBPTR_EL1_PTR_SHIFT 0 #define TRBBASER_EL1_BASE_MASK GENMASK_ULL(63, 12) #define TRBBASER_EL1_BASE_SHIFT 12 #define TRBSR_EL1_EC_MASK GENMASK(31, 26) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index fe2490713754..ec493e43988c 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2273,3 +2273,7 @@ Enum 2:1 FM EndEnum Field 0 E EndSysreg + +Sysreg TRBPTR_EL1 3 0 9 11 1 +Field 63:0 PTR +EndSysreg -- cgit From cbaf0cf005f0de92532b713fd8f7497a129f588b Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:45 +0530 Subject: arm64/sysreg: Convert TRBBASER_EL1 register to automatic generation This converts TRBBASER_EL1 register to automatic generation without causing any functional change. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Reviewed-by: Mark Brown Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-11-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 3 --- arch/arm64/tools/sysreg | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 6b3204fbad22..98aa015f6db8 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -227,14 +227,11 @@ /*** End of Statistical Profiling Extension ***/ -#define SYS_TRBBASER_EL1 sys_reg(3, 0, 9, 11, 2) #define SYS_TRBSR_EL1 sys_reg(3, 0, 9, 11, 3) #define SYS_TRBMAR_EL1 sys_reg(3, 0, 9, 11, 4) #define SYS_TRBTRG_EL1 sys_reg(3, 0, 9, 11, 6) #define SYS_TRBIDR_EL1 sys_reg(3, 0, 9, 11, 7) -#define TRBBASER_EL1_BASE_MASK GENMASK_ULL(63, 12) -#define TRBBASER_EL1_BASE_SHIFT 12 #define TRBSR_EL1_EC_MASK GENMASK(31, 26) #define TRBSR_EL1_EC_SHIFT 26 #define TRBSR_EL1_IRQ BIT(22) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index ec493e43988c..e51eec07e7c3 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2277,3 +2277,8 @@ EndSysreg Sysreg TRBPTR_EL1 3 0 9 11 1 Field 63:0 PTR EndSysreg + +Sysreg TRBBASER_EL1 3 0 9 11 2 +Field 63:12 BASE +Res0 11:0 +EndSysreg -- cgit From 46f3a5b01fd796f657ecbbefff9874e43e172391 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:46 +0530 Subject: arm64/sysreg: Convert TRBSR_EL1 register to automatic generation This converts TRBSR_EL1 register to automatic generation without causing any functional change. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Reviewed-by: Mark Brown Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-12-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 12 ------------ arch/arm64/tools/sysreg | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 98aa015f6db8..8382e46d1f3f 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -227,24 +227,12 @@ /*** End of Statistical Profiling Extension ***/ -#define SYS_TRBSR_EL1 sys_reg(3, 0, 9, 11, 3) #define SYS_TRBMAR_EL1 sys_reg(3, 0, 9, 11, 4) #define SYS_TRBTRG_EL1 sys_reg(3, 0, 9, 11, 6) #define SYS_TRBIDR_EL1 sys_reg(3, 0, 9, 11, 7) -#define TRBSR_EL1_EC_MASK GENMASK(31, 26) -#define TRBSR_EL1_EC_SHIFT 26 -#define TRBSR_EL1_IRQ BIT(22) -#define TRBSR_EL1_TRG BIT(21) -#define TRBSR_EL1_WRAP BIT(20) -#define TRBSR_EL1_EA BIT(18) -#define TRBSR_EL1_S BIT(17) -#define TRBSR_EL1_MSS_MASK GENMASK(15, 0) -#define TRBSR_EL1_MSS_SHIFT 0 #define TRBSR_EL1_BSC_MASK GENMASK(5, 0) #define TRBSR_EL1_BSC_SHIFT 0 -#define TRBSR_EL1_FSC_MASK GENMASK(5, 0) -#define TRBSR_EL1_FSC_SHIFT 0 #define TRBMAR_EL1_SH_MASK GENMASK(9, 8) #define TRBMAR_EL1_SH_SHIFT 8 #define TRBMAR_EL1_Attr_MASK GENMASK(7, 0) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index e51eec07e7c3..a8b1277318b1 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2282,3 +2282,19 @@ Sysreg TRBBASER_EL1 3 0 9 11 2 Field 63:12 BASE Res0 11:0 EndSysreg + +Sysreg TRBSR_EL1 3 0 9 11 3 +Res0 63:56 +Field 55:32 MSS2 +Field 31:26 EC +Res0 25:24 +Field 23 DAT +Field 22 IRQ +Field 21 TRG +Field 20 WRAP +Res0 19 +Field 18 EA +Field 17 S +Res0 16 +Field 15:0 MSS +EndSysreg -- cgit From 3077b1db9d5743c64343f47d88f1da89625c2590 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:47 +0530 Subject: arm64/sysreg: Convert TRBMAR_EL1 register to automatic generation This converts TRBMAR_EL1 register to automatic generation without causing any functional change. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Reviewed-by: Mark Brown Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-13-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 5 ----- arch/arm64/tools/sysreg | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 8382e46d1f3f..8d599f579c81 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -227,16 +227,11 @@ /*** End of Statistical Profiling Extension ***/ -#define SYS_TRBMAR_EL1 sys_reg(3, 0, 9, 11, 4) #define SYS_TRBTRG_EL1 sys_reg(3, 0, 9, 11, 6) #define SYS_TRBIDR_EL1 sys_reg(3, 0, 9, 11, 7) #define TRBSR_EL1_BSC_MASK GENMASK(5, 0) #define TRBSR_EL1_BSC_SHIFT 0 -#define TRBMAR_EL1_SH_MASK GENMASK(9, 8) -#define TRBMAR_EL1_SH_SHIFT 8 -#define TRBMAR_EL1_Attr_MASK GENMASK(7, 0) -#define TRBMAR_EL1_Attr_SHIFT 0 #define TRBTRG_EL1_TRG_MASK GENMASK(31, 0) #define TRBTRG_EL1_TRG_SHIFT 0 #define TRBIDR_EL1_F BIT(5) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index a8b1277318b1..8464d364cba1 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2298,3 +2298,19 @@ Field 17 S Res0 16 Field 15:0 MSS EndSysreg + +Sysreg TRBMAR_EL1 3 0 9 11 4 +Res0 63:12 +Enum 11:10 PAS + 0b00 SECURE + 0b01 NON_SECURE + 0b10 ROOT + 0b11 REALM +EndEnum +Enum 9:8 SH + 0b00 NON_SHAREABLE + 0b10 OUTER_SHAREABLE + 0b11 INNER_SHAREABLE +EndEnum +Field 7:0 Attr +EndSysreg -- cgit From a56035c95ec6b55746528a107a4dcdeb8bac0147 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:48 +0530 Subject: arm64/sysreg: Convert TRBTRG_EL1 register to automatic generation This converts TRBTRG_EL1 register to automatic generation without causing any functional change. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Reviewed-by: Mark Brown Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-14-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 3 --- arch/arm64/tools/sysreg | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 8d599f579c81..d46679105806 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -227,13 +227,10 @@ /*** End of Statistical Profiling Extension ***/ -#define SYS_TRBTRG_EL1 sys_reg(3, 0, 9, 11, 6) #define SYS_TRBIDR_EL1 sys_reg(3, 0, 9, 11, 7) #define TRBSR_EL1_BSC_MASK GENMASK(5, 0) #define TRBSR_EL1_BSC_SHIFT 0 -#define TRBTRG_EL1_TRG_MASK GENMASK(31, 0) -#define TRBTRG_EL1_TRG_SHIFT 0 #define TRBIDR_EL1_F BIT(5) #define TRBIDR_EL1_P BIT(4) #define TRBIDR_EL1_Align_MASK GENMASK(3, 0) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 8464d364cba1..26da20f3ff40 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2314,3 +2314,8 @@ Enum 9:8 SH EndEnum Field 7:0 Attr EndSysreg + +Sysreg TRBTRG_EL1 3 0 9 11 6 +Res0 63:32 +Field 31:0 TRG +EndSysreg -- cgit From f0d4627f645924edd855a3242ce4f2cdc3d61126 Mon Sep 17 00:00:00 2001 From: Anshuman Khandual Date: Wed, 14 Jun 2023 12:29:49 +0530 Subject: arm64/sysreg: Convert TRBIDR_EL1 register to automatic generation This converts TRBIDR_EL1 register to automatic generation without causing any functional change. Cc: Will Deacon Cc: Marc Zyngier Cc: Mark Brown Cc: Rob Herring Cc: Suzuki K Poulose Cc: James Morse Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Reviewed-by: Mark Brown Signed-off-by: Anshuman Khandual Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20230614065949.146187-15-anshuman.khandual@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/sysreg.h | 6 ------ arch/arm64/tools/sysreg | 13 +++++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index d46679105806..0c07b03d511f 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -227,14 +227,8 @@ /*** End of Statistical Profiling Extension ***/ -#define SYS_TRBIDR_EL1 sys_reg(3, 0, 9, 11, 7) - #define TRBSR_EL1_BSC_MASK GENMASK(5, 0) #define TRBSR_EL1_BSC_SHIFT 0 -#define TRBIDR_EL1_F BIT(5) -#define TRBIDR_EL1_P BIT(4) -#define TRBIDR_EL1_Align_MASK GENMASK(3, 0) -#define TRBIDR_EL1_Align_SHIFT 0 #define SYS_PMINTENSET_EL1 sys_reg(3, 0, 9, 14, 1) #define SYS_PMINTENCLR_EL1 sys_reg(3, 0, 9, 14, 2) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 26da20f3ff40..c585725172d5 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2319,3 +2319,16 @@ Sysreg TRBTRG_EL1 3 0 9 11 6 Res0 63:32 Field 31:0 TRG EndSysreg + +Sysreg TRBIDR_EL1 3 0 9 11 7 +Res0 63:12 +Enum 11:8 EA + 0b0000 NON_DESC + 0b0001 IGNORE + 0b0010 SERROR +EndEnum +Res0 7:6 +Field 5 F +Field 4 P +Field 3:0 Align +EndSysreg -- cgit From 601eaec513cc814c3dc6ed504c7c51ce1b80d0ea Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 14 Jun 2023 17:09:35 +0100 Subject: arm64: consolidate rox page protection logic Consolidate the arm64 decision making for the page protections used for executable pages, used by both the trampoline code and the kernel text mapping code. Acked-by: Mark Rutland Signed-off-by: Russell King (Oracle) Link: https://lore.kernel.org/r/E1q9T3v-00EDmW-BH@rmk-PC.armlinux.org.uk Signed-off-by: Catalin Marinas --- arch/arm64/mm/mmu.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index af6bc8403ee4..4829abe017e9 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -663,12 +663,17 @@ static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end, vm_area_add_early(vma); } +static pgprot_t kernel_exec_prot(void) +{ + return rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; +} + #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 static int __init map_entry_trampoline(void) { int i; - pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; + pgprot_t prot = kernel_exec_prot(); phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start); /* The trampoline is always mapped and can therefore be global */ @@ -723,7 +728,7 @@ static void __init map_kernel(pgd_t *pgdp) * mapping to install SW breakpoints. Allow this (only) when * explicitly requested with rodata=off. */ - pgprot_t text_prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; + pgprot_t text_prot = kernel_exec_prot(); /* * If we have a CPU that supports BTI and a kernel built for -- cgit From 137477c8daac19e4dd4489901fe85c8e3602427b Mon Sep 17 00:00:00 2001 From: Jose Marinho Date: Tue, 6 Jun 2023 10:35:26 +0100 Subject: Documentation/arm64: Update ARM and arch reference This patch clarifies that both Armv8 and v9 are in scope, not just Armv8 systems. Also, ARM is re-written as Arm. Cc: Jeremy Linton Cc: James Morse Cc: Rob Herring Cc: Will Deacon Cc: Jonathan Corbet Cc: Hanjun Guo Cc: linux-arm-kernel@lists.infradead.org Cc: linux-doc@vger.kernel.org Cc: linux-acpi@vger.kernel.org Signed-off-by: Jose Marinho Reviewed-by: Samer El-Haj-Mahmoud Reviewed-by: Robin Murphy Acked-by: Sudeep Holla Reviewed-by: Jeremy Linton Reviewed-by: Hanjun Guo Link: https://lore.kernel.org/r/20230606093528.1494344-2-jose.marinho@arm.com Signed-off-by: Catalin Marinas --- Documentation/arm64/arm-acpi.rst | 41 ++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/Documentation/arm64/arm-acpi.rst b/Documentation/arm64/arm-acpi.rst index 47ecb9930dde..b7c9cc277890 100644 --- a/Documentation/arm64/arm-acpi.rst +++ b/Documentation/arm64/arm-acpi.rst @@ -1,40 +1,41 @@ -===================== -ACPI on ARMv8 Servers -===================== +=================== +ACPI on Arm systems +=================== -ACPI can be used for ARMv8 general purpose servers designed to follow -the ARM SBSA (Server Base System Architecture) [0] and SBBR (Server +ACPI can be used for Armv8 and Armv9 systems designed to follow +the Arm SBSA (Server Base System Architecture) [0] and SBBR (Server Base Boot Requirements) [1] specifications. Please note that the SBBR can be retrieved simply by visiting [1], but the SBSA is currently only available to those with an ARM login due to ARM IP licensing concerns. -The ARMv8 kernel implements the reduced hardware model of ACPI version + +The Arm kernel implements the reduced hardware model of ACPI version 5.1 or later. Links to the specification and all external documents it refers to are managed by the UEFI Forum. The specification is available at http://www.uefi.org/specifications and documents referenced by the specification can be found via http://www.uefi.org/acpi. -If an ARMv8 system does not meet the requirements of the SBSA and SBBR, +If an Arm system does not meet the requirements of the BSA and BBR, or cannot be described using the mechanisms defined in the required ACPI specifications, then ACPI may not be a good fit for the hardware. While the documents mentioned above set out the requirements for building -industry-standard ARMv8 servers, they also apply to more than one operating +industry-standard Arm systems, they also apply to more than one operating system. The purpose of this document is to describe the interaction between -ACPI and Linux only, on an ARMv8 system -- that is, what Linux expects of +ACPI and Linux only, on an Arm system -- that is, what Linux expects of ACPI and what ACPI can expect of Linux. -Why ACPI on ARM? +Why ACPI on Arm? ---------------- Before examining the details of the interface between ACPI and Linux, it is useful to understand why ACPI is being used. Several technologies already exist in Linux for describing non-enumerable hardware, after all. In this section we summarize a blog post [2] from Grant Likely that outlines the -reasoning behind ACPI on ARMv8 servers. Actually, we snitch a good portion +reasoning behind ACPI on Arm systems. Actually, we snitch a good portion of the summary text almost directly, to be honest. -The short form of the rationale for ACPI on ARM is: +The short form of the rationale for ACPI on Arm is: - ACPI’s byte code (AML) allows the platform to encode hardware behavior, while DT explicitly does not support this. For hardware vendors, being @@ -47,7 +48,7 @@ The short form of the rationale for ACPI on ARM is: - In the enterprise server environment, ACPI has established bindings (such as for RAS) which are currently used in production systems. DT does not. - Such bindings could be defined in DT at some point, but doing so means ARM + Such bindings could be defined in DT at some point, but doing so means Arm and x86 would end up using completely different code paths in both firmware and the kernel. @@ -108,7 +109,7 @@ recent version of the kernel. Relationship with Device Tree ----------------------------- -ACPI support in drivers and subsystems for ARMv8 should never be mutually +ACPI support in drivers and subsystems for Arm should never be mutually exclusive with DT support at compile time. At boot time the kernel will only use one description method depending on @@ -121,11 +122,11 @@ time). Booting using ACPI tables ------------------------- -The only defined method for passing ACPI tables to the kernel on ARMv8 +The only defined method for passing ACPI tables to the kernel on Arm is via the UEFI system configuration table. Just so it is explicit, this means that ACPI is only supported on platforms that boot via UEFI. -When an ARMv8 system boots, it can either have DT information, ACPI tables, +When an Arm system boots, it can either have DT information, ACPI tables, or in some very unusual cases, both. If no command line parameters are used, the kernel will try to use DT for device enumeration; if there is no DT present, the kernel will try to use ACPI tables, but only if they are present. @@ -448,7 +449,7 @@ ASWG ---- The ACPI specification changes regularly. During the year 2014, for instance, version 5.1 was released and version 6.0 substantially completed, with most of -the changes being driven by ARM-specific requirements. Proposed changes are +the changes being driven by Arm-specific requirements. Proposed changes are presented and discussed in the ASWG (ACPI Specification Working Group) which is a part of the UEFI Forum. The current version of the ACPI specification is 6.1 release in January 2016. @@ -456,7 +457,7 @@ is 6.1 release in January 2016. Participation in this group is open to all UEFI members. Please see http://www.uefi.org/workinggroup for details on group membership. -It is the intent of the ARMv8 ACPI kernel code to follow the ACPI specification +It is the intent of the Arm ACPI kernel code to follow the ACPI specification as closely as possible, and to only implement functionality that complies with the released standards from UEFI ASWG. As a practical matter, there will be vendors that provide bad ACPI tables or violate the standards in some way. @@ -470,12 +471,12 @@ likely be willing to assist in submitting ECRs. Linux Code ---------- -Individual items specific to Linux on ARM, contained in the Linux +Individual items specific to Linux on Arm, contained in the Linux source code, are in the list that follows: ACPI_OS_NAME This macro defines the string to be returned when - an ACPI method invokes the _OS method. On ARM64 + an ACPI method invokes the _OS method. On Arm systems, this macro will be "Linux" by default. The command line parameter acpi_os= can be used to set it to some other value. The -- cgit From 8c350dfc90656639e9482e8a17625e4233d4a23f Mon Sep 17 00:00:00 2001 From: Jose Marinho Date: Tue, 6 Jun 2023 10:35:27 +0100 Subject: Documentation/arm64: Update references in arm-acpi This patch: - Updates the reference to the DSD document, - Removes the unused reference to AMD Seattle, - Updates the references to BBR, BSA and SBSA. Cc: Jeremy Linton Cc: James Morse Cc: Rob Herring Cc: Will Deacon Cc: Jonathan Corbet Cc: Hanjun Guo Cc: linux-arm-kernel@lists.infradead.org Cc: linux-doc@vger.kernel.org Cc: linux-acpi@vger.kernel.org Signed-off-by: Jose Marinho Reviewed-by: Samer El-Haj-Mahmoud Reviewed-by: Robin Murphy Acked-by: Sudeep Holla Reviewed-by: Jeremy Linton Reviewed-by: Hanjun Guo Link: https://lore.kernel.org/r/20230606093528.1494344-3-jose.marinho@arm.com Signed-off-by: Catalin Marinas --- Documentation/arm64/arm-acpi.rst | 56 +++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/Documentation/arm64/arm-acpi.rst b/Documentation/arm64/arm-acpi.rst index b7c9cc277890..97f547b19774 100644 --- a/Documentation/arm64/arm-acpi.rst +++ b/Documentation/arm64/arm-acpi.rst @@ -3,11 +3,11 @@ ACPI on Arm systems =================== ACPI can be used for Armv8 and Armv9 systems designed to follow -the Arm SBSA (Server Base System Architecture) [0] and SBBR (Server -Base Boot Requirements) [1] specifications. Please note that the SBBR -can be retrieved simply by visiting [1], but the SBSA is currently only -available to those with an ARM login due to ARM IP licensing concerns. - +the BSA (Arm Base System Architecture) [0] and BBR (Arm +Base Boot Requirements) [1] specifications. Both BSA and BBR are publicly +accessible documents. +Arm Servers, in addition to being BSA compliant, comply with a set +of rules defined in SBSA (Server Base System Architecture) [2]. The Arm kernel implements the reduced hardware model of ACPI version 5.1 or later. Links to the specification and all external documents @@ -31,7 +31,7 @@ Why ACPI on Arm? Before examining the details of the interface between ACPI and Linux, it is useful to understand why ACPI is being used. Several technologies already exist in Linux for describing non-enumerable hardware, after all. In this -section we summarize a blog post [2] from Grant Likely that outlines the +section we summarize a blog post [3] from Grant Likely that outlines the reasoning behind ACPI on Arm systems. Actually, we snitch a good portion of the summary text almost directly, to be honest. @@ -270,16 +270,14 @@ Drivers should look for device properties in the _DSD object ONLY; the _DSD object is described in the ACPI specification section 6.2.5, but this only describes how to define the structure of an object returned via _DSD, and how specific data structures are defined by specific UUIDs. Linux should -only use the _DSD Device Properties UUID [5]: +only use the _DSD Device Properties UUID [4]: - UUID: daffd814-6eba-4d8c-8a91-bc9bbf4aa301 - - https://www.uefi.org/sites/default/files/resources/_DSD-device-properties-UUID.pdf - -The UEFI Forum provides a mechanism for registering device properties [4] -so that they may be used across all operating systems supporting ACPI. -Device properties that have not been registered with the UEFI Forum should -not be used. +Common device properties can be registered by creating a pull request to [4] so +that they may be used across all operating systems supporting ACPI. +Device properties that have not been registered with the UEFI Forum can be used +but not as "uefi-" common properties. Before creating new device properties, check to be sure that they have not been defined before and either registered in the Linux kernel documentation @@ -307,7 +305,7 @@ process. Once registration and review have been completed, the kernel provides an interface for looking up device properties in a manner independent of -whether DT or ACPI is being used. This API should be used [6]; it can +whether DT or ACPI is being used. This API should be used [5]; it can eliminate some duplication of code paths in driver probing functions and discourage divergence between DT bindings and ACPI device properties. @@ -491,31 +489,23 @@ Documentation/arm64/acpi_object_usage.rst. References ---------- -[0] http://silver.arm.com - document ARM-DEN-0029, or newer: - "Server Base System Architecture", version 2.3, dated 27 Mar 2014 +[0] https://developer.arm.com/documentation/den0094/latest + document Arm-DEN-0094: "Arm Base System Architecture", version 1.0C, dated 6 Oct 2022 + +[1] https://developer.arm.com/documentation/den0044/latest + Document Arm-DEN-0044: "Arm Base Boot Requirements", version 2.0G, dated 15 Apr 2022 -[1] http://infocenter.arm.com/help/topic/com.arm.doc.den0044a/Server_Base_Boot_Requirements.pdf - Document ARM-DEN-0044A, or newer: "Server Base Boot Requirements, System - Software on ARM Platforms", dated 16 Aug 2014 +[2] https://developer.arm.com/documentation/den0029/latest + Document Arm-DEN-0029: "Arm Server Base System Architecture", version 7.1, dated 06 Oct 2022 -[2] http://www.secretlab.ca/archives/151, +[3] http://www.secretlab.ca/archives/151, 10 Jan 2015, Copyright (c) 2015, Linaro Ltd., written by Grant Likely. -[3] AMD ACPI for Seattle platform documentation - http://amd-dev.wpengine.netdna-cdn.com/wordpress/media/2012/10/Seattle_ACPI_Guide.pdf - - -[4] http://www.uefi.org/acpi - please see the link for the "ACPI _DSD Device - Property Registry Instructions" - -[5] http://www.uefi.org/acpi - please see the link for the "_DSD (Device - Specific Data) Implementation Guide" +[4] _DSD (Device Specific Data) Implementation Guide + https://github.com/UEFI/DSD-Guide/blob/main/dsd-guide.pdf -[6] Kernel code for the unified device +[5] Kernel code for the unified device property interface can be found in include/linux/property.h and drivers/base/property.c. -- cgit From 3927eaff464f1fd6eb49388b4c3eb1df677b0360 Mon Sep 17 00:00:00 2001 From: Jose Marinho Date: Tue, 6 Jun 2023 10:35:28 +0100 Subject: Documentation/arm64: Update ACPI tables from BBR The BBR specification requires (or conditionally requires) a set of ACPI tables for a proper working system. This commit updates: - the list of ACPI tables to reflect the contents of BBR version 2.0 (see https://developer.arm.com/documentation/den0044/g). - the list of ACPI tables in acpi_object_usage. This last update ensures that both files remain coherent. Cc: Jeremy Linton Cc: James Morse Cc: Rob Herring Cc: Will Deacon Cc: Jonathan Corbet Cc: Hanjun Guo Cc: linux-arm-kernel@lists.infradead.org Cc: linux-doc@vger.kernel.org Cc: linux-acpi@vger.kernel.org Signed-off-by: Jose Marinho Reviewed-by: Samer El-Haj-Mahmoud Acked-by: Sudeep Holla Reviewed-by: Jeremy Linton Reviewed-by: Hanjun Guo Link: https://lore.kernel.org/r/20230606093528.1494344-4-jose.marinho@arm.com Signed-off-by: Catalin Marinas --- Documentation/arm64/acpi_object_usage.rst | 81 +++++++++++++++++++++++++++++-- Documentation/arm64/arm-acpi.rst | 72 ++++++++++++++++++++++++--- 2 files changed, 140 insertions(+), 13 deletions(-) diff --git a/Documentation/arm64/acpi_object_usage.rst b/Documentation/arm64/acpi_object_usage.rst index 484ef9676653..1da22200fdf8 100644 --- a/Documentation/arm64/acpi_object_usage.rst +++ b/Documentation/arm64/acpi_object_usage.rst @@ -17,16 +17,37 @@ For ACPI on arm64, tables also fall into the following categories: - Recommended: BERT, EINJ, ERST, HEST, PCCT, SSDT - - Optional: BGRT, CPEP, CSRT, DBG2, DRTM, ECDT, FACS, FPDT, IBFT, - IORT, MCHI, MPST, MSCT, NFIT, PMTT, RASF, SBST, SLIT, SPMI, SRAT, - STAO, TCPA, TPM2, UEFI, XENV + - Optional: AGDI, BGRT, CEDT, CPEP, CSRT, DBG2, DRTM, ECDT, FACS, FPDT, + HMAT, IBFT, IORT, MCHI, MPAM, MPST, MSCT, NFIT, PMTT, PPTT, RASF, SBST, + SDEI, SLIT, SPMI, SRAT, STAO, TCPA, TPM2, UEFI, XENV - - Not supported: BOOT, DBGP, DMAR, ETDT, HPET, IVRS, LPIT, MSDM, OEMx, - PSDT, RSDT, SLIC, WAET, WDAT, WDRT, WPBT + - Not supported: AEST, APMT, BOOT, DBGP, DMAR, ETDT, HPET, IVRS, LPIT, + MSDM, OEMx, PDTT, PSDT, RAS2, RSDT, SLIC, WAET, WDAT, WDRT, WPBT ====== ======================================================================== Table Usage for ARMv8 Linux ====== ======================================================================== +AEST Signature Reserved (signature == "AEST") + + **Arm Error Source Table** + + This table informs the OS of any error nodes in the system that are + compliant with the Arm RAS architecture. + +AGDI Signature Reserved (signature == "AGDI") + + **Arm Generic diagnostic Dump and Reset Device Interface Table** + + This table describes a non-maskable event, that is used by the platform + firmware, to request the OS to generate a diagnostic dump and reset the device. + +APMT Signature Reserved (signature == "APMT") + + **Arm Performance Monitoring Table** + + This table describes the properties of PMU support implmented by + components in the system. + BERT Section 18.3 (signature == "BERT") **Boot Error Record Table** @@ -47,6 +68,13 @@ BGRT Section 5.2.22 (signature == "BGRT") Optional, not currently supported, with no real use-case for an ARM server. +CEDT Signature Reserved (signature == "CEDT") + + **CXL Early Discovery Table** + + This table allows the OS to discover any CXL Host Bridges and the Host + Bridge registers. + CPEP Section 5.2.18 (signature == "CPEP") **Corrected Platform Error Polling table** @@ -184,6 +212,15 @@ HEST Section 18.3.2 (signature == "HEST") Must be supplied if RAS support is provided by the platform. It is recommended this table be supplied. +HMAT Section 5.2.28 (signature == "HMAT") + + **Heterogeneous Memory Attribute Table** + + This table describes the memory attributes, such as memory side cache + attributes and bandwidth and latency details, related to Memory Proximity + Domains. The OS uses this information to optimize the system memory + configuration. + HPET Signature Reserved (signature == "HPET") **High Precision Event timer Table** @@ -241,6 +278,13 @@ MCHI Signature Reserved (signature == "MCHI") Optional, not currently supported. +MPAM Signature Reserved (signature == "MPAM") + + **Memory Partitioning And Monitoring table** + + This table allows the OS to discover the MPAM controls implemented by + the subsystems. + MPST Section 5.2.21 (signature == "MPST") **Memory Power State Table** @@ -281,18 +325,39 @@ PCCT Section 14.1 (signature == "PCCT) Recommend for use on arm64; use of PCC is recommended when using CPPC to control performance and power for platform processors. +PDTT Section 5.2.29 (signature == "PDTT") + + **Platform Debug Trigger Table** + + This table describes PCC channels used to gather debug logs of + non-architectural features. + + PMTT Section 5.2.21.12 (signature == "PMTT") **Platform Memory Topology Table** Optional, not currently supported. +PPTT Section 5.2.30 (signature == "PPTT") + + **Processor Properties Topology Table** + + This table provides the processor and cache topology. + PSDT Section 5.2.11.3 (signature == "PSDT") **Persistent System Description Table** Obsolete table, will not be supported. +RAS2 Section 5.2.21 (signature == "RAS2") + + **RAS Features 2 table** + + This table provides interfaces for the RAS capabilities implemented in + the platform. + RASF Section 5.2.20 (signature == "RASF") **RAS Feature table** @@ -318,6 +383,12 @@ SBST Section 5.2.14 (signature == "SBST") Optional, not currently supported. +SDEI Signature Reserved (signature == "SDEI") + + **Software Delegated Exception Interface table** + + This table advertises the presence of the SDEI interface. + SLIC Signature Reserved (signature == "SLIC") **Software LIcensing table** diff --git a/Documentation/arm64/arm-acpi.rst b/Documentation/arm64/arm-acpi.rst index 97f547b19774..37ec5e9b1575 100644 --- a/Documentation/arm64/arm-acpi.rst +++ b/Documentation/arm64/arm-acpi.rst @@ -170,7 +170,7 @@ hardware reduced mode must be set to zero. For the ACPI core to operate properly, and in turn provide the information the kernel needs to configure devices, it expects to find the following -tables (all section numbers refer to the ACPI 6.1 specification): +tables (all section numbers refer to the ACPI 6.5 specification): - RSDP (Root System Description Pointer), section 5.2.5 @@ -185,20 +185,76 @@ tables (all section numbers refer to the ACPI 6.1 specification): - GTDT (Generic Timer Description Table), section 5.2.24 + - PPTT (Processor Properties Topology Table), section 5.2.30 + + - DBG2 (DeBuG port table 2), section 5.2.6, specifically Table 5-6. + + - APMT (Arm Performance Monitoring unit Table), section 5.2.6, specifically Table 5-6. + + - AGDI (Arm Generic diagnostic Dump and Reset Device Interface Table), section 5.2.6, specifically Table 5-6. + - If PCI is supported, the MCFG (Memory mapped ConFiGuration - Table), section 5.2.6, specifically Table 5-31. + Table), section 5.2.6, specifically Table 5-6. - If booting without a console= kernel parameter is supported, the SPCR (Serial Port Console Redirection table), - section 5.2.6, specifically Table 5-31. + section 5.2.6, specifically Table 5-6. - If necessary to describe the I/O topology, SMMUs and GIC ITSs, the IORT (Input Output Remapping Table, section 5.2.6, specifically - Table 5-31). + Table 5-6). + + - If NUMA is supported, the following tables are required: + + - SRAT (System Resource Affinity Table), section 5.2.16 + + - SLIT (System Locality distance Information Table), section 5.2.17 + + - If NUMA is supported, and the system contains heterogeneous memory, + the HMAT (Heterogeneous Memory Attribute Table), section 5.2.28. + + - If the ACPI Platform Error Interfaces are required, the following + tables are conditionally required: + + - BERT (Boot Error Record Table, section 18.3.1) + + - EINJ (Error INJection table, section 18.6.1) + + - ERST (Error Record Serialization Table, section 18.5) + + - HEST (Hardware Error Source Table, section 18.3.2) + + - SDEI (Software Delegated Exception Interface table, section 5.2.6, + specifically Table 5-6) + + - AEST (Arm Error Source Table, section 5.2.6, + specifically Table 5-6) + + - RAS2 (ACPI RAS2 feature table, section 5.2.21) + + - If the system contains controllers using PCC channel, the + PCCT (Platform Communications Channel Table), section 14.1 + + - If the system contains a controller to capture board-level system state, + and communicates with the host via PCC, the PDTT (Platform Debug Trigger + Table), section 5.2.29. + + - If NVDIMM is supported, the NFIT (NVDIMM Firmware Interface Table), section 5.2.26 + + - If video framebuffer is present, the BGRT (Boot Graphics Resource Table), section 5.2.23 + + - If IPMI is implemented, the SPMI (Server Platform Management Interface), + section 5.2.6, specifically Table 5-6. + + - If the system contains a CXL Host Bridge, the CEDT (CXL Early Discovery + Table), section 5.2.6, specifically Table 5-6. + + - If the system supports MPAM, the MPAM (Memory Partitioning And Monitoring table), section 5.2.6, + specifically Table 5-6. + + - If the system lacks persistent storage, the IBFT (ISCSI Boot Firmware + Table), section 5.2.6, specifically Table 5-6. - - If NUMA is supported, the SRAT (System Resource Affinity Table) - and SLIT (System Locality distance Information Table), sections - 5.2.16 and 5.2.17, respectively. If the above tables are not all present, the kernel may or may not be able to boot properly since it may not be able to configure all of the @@ -450,7 +506,7 @@ version 5.1 was released and version 6.0 substantially completed, with most of the changes being driven by Arm-specific requirements. Proposed changes are presented and discussed in the ASWG (ACPI Specification Working Group) which is a part of the UEFI Forum. The current version of the ACPI specification -is 6.1 release in January 2016. +is 6.5 release in August 2022. Participation in this group is open to all UEFI members. Please see http://www.uefi.org/workinggroup for details on group membership. -- cgit From b9293d457ff3de415fa07d7e35978bceb29c3827 Mon Sep 17 00:00:00 2001 From: Jamie Iles Date: Tue, 13 Jun 2023 15:19:59 +0100 Subject: arm64/mm: remove now-superfluous ISBs from TTBR writes At the time of authoring 7655abb95386 ("arm64: mm: Move ASID from TTBR0 to TTBR1"), the Arm ARM did not specify any ordering guarantees for direct writes to TTBR0_ELx and TTBR1_ELx and so an ISB was required after each write to ensure TLBs would only be populated from the expected (or reserved tables). In a recent update to the Arm ARM, the requirements have been relaxed to reflect the implementation of current CPUs and required implementation of future CPUs to read (RDYDPX in D8.2.3 Translation table base address register): Direct writes to TTBR0_ELx and TTBR1_ELx occur in program order relative to one another, without the need for explicit synchronization. For any one translation, all indirect reads of TTBR0_ELx and TTBR1_ELx that are made as part of the translation observe only one point in that order of direct writes. Remove the superfluous ISBs to optimize uaccess helpers and context switch. Cc: Will Deacon Cc: Mark Rutland Signed-off-by: Jamie Iles Reviewed-by: Mark Rutland Link: https://lore.kernel.org/r/20230613141959.92697-1-quic_jiles@quicinc.com [catalin.marinas@arm.com: rename __cpu_set_reserved_ttbr0 to ..._nosync] [catalin.marinas@arm.com: move the cpu_set_reserved_ttbr0_nosync() call to cpu_do_switch_mm()] Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/asm-uaccess.h | 2 -- arch/arm64/include/asm/mmu_context.h | 8 ++++++-- arch/arm64/include/asm/uaccess.h | 2 -- arch/arm64/mm/context.c | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h index 75b211c98dea..5b6efe8abeeb 100644 --- a/arch/arm64/include/asm/asm-uaccess.h +++ b/arch/arm64/include/asm/asm-uaccess.h @@ -18,7 +18,6 @@ bic \tmp1, \tmp1, #TTBR_ASID_MASK sub \tmp1, \tmp1, #RESERVED_SWAPPER_OFFSET // reserved_pg_dir msr ttbr0_el1, \tmp1 // set reserved TTBR0_EL1 - isb add \tmp1, \tmp1, #RESERVED_SWAPPER_OFFSET msr ttbr1_el1, \tmp1 // set reserved ASID isb @@ -31,7 +30,6 @@ extr \tmp2, \tmp2, \tmp1, #48 ror \tmp2, \tmp2, #16 msr ttbr1_el1, \tmp2 // set the active ASID - isb msr ttbr0_el1, \tmp1 // set the non-PAN TTBR0_EL1 isb .endm diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 56911691bef0..86f6870c64a6 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -39,11 +39,16 @@ static inline void contextidr_thread_switch(struct task_struct *next) /* * Set TTBR0 to reserved_pg_dir. No translations will be possible via TTBR0. */ -static inline void cpu_set_reserved_ttbr0(void) +static inline void cpu_set_reserved_ttbr0_nosync(void) { unsigned long ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); write_sysreg(ttbr, ttbr0_el1); +} + +static inline void cpu_set_reserved_ttbr0(void) +{ + cpu_set_reserved_ttbr0_nosync(); isb(); } @@ -52,7 +57,6 @@ void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) { BUG_ON(pgd == swapper_pg_dir); - cpu_set_reserved_ttbr0(); cpu_do_switch_mm(virt_to_phys(pgd),mm); } diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 05f4fc265428..14be5000c5a0 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -65,7 +65,6 @@ static inline void __uaccess_ttbr0_disable(void) ttbr &= ~TTBR_ASID_MASK; /* reserved_pg_dir placed before swapper_pg_dir */ write_sysreg(ttbr - RESERVED_SWAPPER_OFFSET, ttbr0_el1); - isb(); /* Set reserved ASID */ write_sysreg(ttbr, ttbr1_el1); isb(); @@ -89,7 +88,6 @@ static inline void __uaccess_ttbr0_enable(void) ttbr1 &= ~TTBR_ASID_MASK; /* safety measure */ ttbr1 |= ttbr0 & TTBR_ASID_MASK; write_sysreg(ttbr1, ttbr1_el1); - isb(); /* Restore user page table */ write_sysreg(ttbr0, ttbr0_el1); diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index e1e0dca01839..188197590fc9 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -364,8 +364,8 @@ void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) ttbr1 &= ~TTBR_ASID_MASK; ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); + cpu_set_reserved_ttbr0_nosync(); write_sysreg(ttbr1, ttbr1_el1); - isb(); write_sysreg(ttbr0, ttbr0_el1); isb(); post_ttbr_update_workaround(); -- cgit From ab9b4008092c86dc12497af155a0901cc1156999 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 15 Jun 2023 11:26:28 +0100 Subject: arm64: mm: fix VA-range sanity check Both create_mapping_noalloc() and update_mapping_prot() sanity-check their 'virt' parameter, but the check itself doesn't make much sense. The condition used today appears to be a historical accident. The sanity-check condition: if ((virt >= PAGE_END) && (virt < VMALLOC_START)) { [ ... warning here ... ] return; } ... can only be true for the KASAN shadow region or the module region, and there's no reason to exclude these specifically for creating and updateing mappings. When arm64 support was first upstreamed in commit: c1cc1552616d0f35 ("arm64: MMU initialisation") ... the condition was: if (virt < VMALLOC_START) { [ ... warning here ... ] return; } At the time, VMALLOC_START was the lowest kernel address, and this was checking whether 'virt' would be translated via TTBR1. Subsequently in commit: 14c127c957c1c607 ("arm64: mm: Flip kernel VA space") ... the condition was changed to: if ((virt >= VA_START) && (virt < VMALLOC_START)) { [ ... warning here ... ] return; } This appear to have been a thinko. The commit moved the linear map to the bottom of the kernel address space, with VMALLOC_START being at the halfway point. The old condition would warn for changes to the linear map below this, and at the time VA_START was the end of the linear map. Subsequently we cleaned up the naming of VA_START in commit: 77ad4ce69321abbe ("arm64: memory: rename VA_START to PAGE_END") ... keeping the erroneous condition as: if ((virt >= PAGE_END) && (virt < VMALLOC_START)) { [ ... warning here ... ] return; } Correct the condition to check against the start of the TTBR1 address space, which is currently PAGE_OFFSET. This simplifies the logic, and more clearly matches the "outside kernel range" message in the warning. Signed-off-by: Mark Rutland Cc: Russell King Cc: Steve Capper Cc: Will Deacon Reviewed-by: Russell King (Oracle) Link: https://lore.kernel.org/r/20230615102628.1052103-1-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/mm/mmu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 4829abe017e9..95d360805f8a 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -451,7 +451,7 @@ static phys_addr_t pgd_pgtable_alloc(int shift) void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot) { - if ((virt >= PAGE_END) && (virt < VMALLOC_START)) { + if (virt < PAGE_OFFSET) { pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", &phys, virt); return; @@ -478,7 +478,7 @@ void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, static void update_mapping_prot(phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot) { - if ((virt >= PAGE_END) && (virt < VMALLOC_START)) { + if (virt < PAGE_OFFSET) { pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n", &phys, virt); return; -- cgit From af3215fd02308c9aa982a028284d6414eeb60c04 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Fri, 9 Jun 2023 23:43:41 +0100 Subject: arm64/fpsimd: Exit streaming mode when flushing tasks Ensure there is no path where we might attempt to save SME state after we flush a task by updating the SVCR register state as well as updating our in memory state. I haven't seen a specific case where this is happening or seen a path where it might happen but for the cost of a single low overhead instruction it seems sensible to close the potential gap. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20230607-arm64-flush-svcr-v2-1-827306001841@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/kernel/fpsimd.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 2fbafa5cc7ac..7a1aeb95d7c3 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -1649,6 +1649,7 @@ void fpsimd_flush_thread(void) fpsimd_flush_thread_vl(ARM64_VEC_SME); current->thread.svcr = 0; + sme_smstop(); } current->thread.fp_type = FP_STATE_FPSIMD; -- cgit From 833b0f07b915f40db352063df0e13bc91fb0f42f Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 21 Jun 2023 02:36:06 +0100 Subject: kselftest/arm64: Log signal code and address for unexpected signals If we get an unexpected signal during a signal test log a bit more data to aid diagnostics. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20230620-arm64-selftest-log-wrong-signal-v1-1-3fe29bdaaf38@kernel.org Signed-off-by: Catalin Marinas --- tools/testing/selftests/arm64/signal/test_signals_utils.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/arm64/signal/test_signals_utils.c b/tools/testing/selftests/arm64/signal/test_signals_utils.c index 40be8443949d..0dc948db3a4a 100644 --- a/tools/testing/selftests/arm64/signal/test_signals_utils.c +++ b/tools/testing/selftests/arm64/signal/test_signals_utils.c @@ -249,7 +249,8 @@ static void default_handler(int signum, siginfo_t *si, void *uc) fprintf(stderr, "-- Timeout !\n"); } else { fprintf(stderr, - "-- RX UNEXPECTED SIGNAL: %d\n", signum); + "-- RX UNEXPECTED SIGNAL: %d code %d address %p\n", + signum, si->si_code, si->si_addr); } default_result(current, 1); } -- cgit From 615af0021a612ea66312a5deddd39cc0d8b70e78 Mon Sep 17 00:00:00 2001 From: Song Shuai Date: Fri, 9 Jun 2023 15:50:47 +0800 Subject: arm64: hibernate: remove WARN_ON in save_processor_state During hibernation or restoration, freeze_secondary_cpus checks num_online_cpus via BUG_ON, and the subsequent save_processor_state also does the checking with WARN_ON. In the case of CONFIG_PM_SLEEP_SMP=n, freeze_secondary_cpus is not defined, but the sole possible condition to disable CONFIG_PM_SLEEP_SMP is !SMP where num_online_cpus is always 1. We also don't have to check it in save_processor_state. So remove the unnecessary checking in save_processor_state. Signed-off-by: Song Shuai Acked-by: Will Deacon Link: https://lore.kernel.org/r/20230609075049.2651723-3-songshuaishuai@tinylab.org Signed-off-by: Catalin Marinas --- arch/arm64/kernel/hibernate.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 788597a6b6a2..02870beb271e 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -99,7 +99,6 @@ int pfn_is_nosave(unsigned long pfn) void notrace save_processor_state(void) { - WARN_ON(num_online_cpus() != 1); } void notrace restore_processor_state(void) -- cgit From a0238ada560fce7fa1b5fb3ace60c0a575d3131a Mon Sep 17 00:00:00 2001 From: Chaitanya S Prakash Date: Mon, 19 Jun 2023 14:08:02 +0530 Subject: Documentation/arm64: Add ptdump documentation ptdump is a debugfs interface used to dump the kernel page tables. It provides a comprehensive overview about the kernel's virtual memory layout, page table entries and associated page attributes. A document detailing how to enable ptdump in the kernel and analyse its output has been added. Changes in V2: - Corrected command to cat /sys/kernel/debug/kernel_page_tables Changes in V1: https://lore.kernel.org/all/20230613064845.1882177-1-chaitanyas.prakash@arm.com/ Cc: Will Deacon Cc: Jonathan Corbet CC: linux-arm-kernel@lists.infradead.org Cc: linux-doc@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Chaitanya S Prakash Link: https://lore.kernel.org/r/20230619083802.76092-1-chaitanyas.prakash@arm.com [catalin.marinas@arm.com: various minor fixups; sorted index.rst in alphabetical order] Signed-off-by: Catalin Marinas --- Documentation/arm64/index.rst | 1 + Documentation/arm64/ptdump.rst | 96 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 Documentation/arm64/ptdump.rst diff --git a/Documentation/arm64/index.rst b/Documentation/arm64/index.rst index ae21f8118830..08f80730d4ed 100644 --- a/Documentation/arm64/index.rst +++ b/Documentation/arm64/index.rst @@ -20,6 +20,7 @@ ARM64 Architecture memory-tagging-extension perf pointer-authentication + ptdump silicon-errata sme sve diff --git a/Documentation/arm64/ptdump.rst b/Documentation/arm64/ptdump.rst new file mode 100644 index 000000000000..5dcfc5d7cddf --- /dev/null +++ b/Documentation/arm64/ptdump.rst @@ -0,0 +1,96 @@ +====================== +Kernel page table dump +====================== + +ptdump is a debugfs interface that provides a detailed dump of the +kernel page tables. It offers a comprehensive overview of the kernel +virtual memory layout as well as the attributes associated with the +various regions in a human-readable format. It is useful to dump the +kernel page tables to verify permissions and memory types. Examining the +page table entries and permissions helps identify potential security +vulnerabilities such as mappings with overly permissive access rights or +improper memory protections. + +Memory hotplug allows dynamic expansion or contraction of available +memory without requiring a system reboot. To maintain the consistency +and integrity of the memory management data structures, arm64 makes use +of the ``mem_hotplug_lock`` semaphore in write mode. Additionally, in +read mode, ``mem_hotplug_lock`` supports an efficient implementation of +``get_online_mems()`` and ``put_online_mems()``. These protect the +offlining of memory being accessed by the ptdump code. + +In order to dump the kernel page tables, enable the following +configurations and mount debugfs:: + + CONFIG_GENERIC_PTDUMP=y + CONFIG_PTDUMP_CORE=y + CONFIG_PTDUMP_DEBUGFS=y + + mount -t debugfs nodev /sys/kernel/debug + cat /sys/kernel/debug/kernel_page_tables + +On analysing the output of ``cat /sys/kernel/debug/kernel_page_tables`` +one can derive information about the virtual address range of the entry, +followed by size of the memory region covered by this entry, the +hierarchical structure of the page tables and finally the attributes +associated with each page. The page attributes provide information about +access permissions, execution capability, type of mapping such as leaf +level PTE or block level PGD, PMD and PUD, and access status of a page +within the kernel memory. Assessing these attributes can assist in +understanding the memory layout, access patterns and security +characteristics of the kernel pages. + +Kernel virtual memory layout example:: + + start address end address size attributes + +---------------------------------------------------------------------------------------+ + | ---[ Linear Mapping start ]---------------------------------------------------------- | + | .................. | + | 0xfff0000000000000-0xfff0000000210000 2112K PTE RW NX SHD AF UXN MEM/NORMAL-TAGGED | + | 0xfff0000000210000-0xfff0000001c00000 26560K PTE ro NX SHD AF UXN MEM/NORMAL | + | .................. | + | ---[ Linear Mapping end ]------------------------------------------------------------ | + +---------------------------------------------------------------------------------------+ + | ---[ Modules start ]----------------------------------------------------------------- | + | .................. | + | 0xffff800000000000-0xffff800008000000 128M PTE | + | .................. | + | ---[ Modules end ]------------------------------------------------------------------- | + +---------------------------------------------------------------------------------------+ + | ---[ vmalloc() area ]---------------------------------------------------------------- | + | .................. | + | 0xffff800008010000-0xffff800008200000 1984K PTE ro x SHD AF UXN MEM/NORMAL | + | 0xffff800008200000-0xffff800008e00000 12M PTE ro x SHD AF CON UXN MEM/NORMAL | + | .................. | + | ---[ vmalloc() end ]----------------------------------------------------------------- | + +---------------------------------------------------------------------------------------+ + | ---[ Fixmap start ]------------------------------------------------------------------ | + | .................. | + | 0xfffffbfffdb80000-0xfffffbfffdb90000 64K PTE ro x SHD AF UXN MEM/NORMAL | + | 0xfffffbfffdb90000-0xfffffbfffdba0000 64K PTE ro NX SHD AF UXN MEM/NORMAL | + | .................. | + | ---[ Fixmap end ]-------------------------------------------------------------------- | + +---------------------------------------------------------------------------------------+ + | ---[ PCI I/O start ]----------------------------------------------------------------- | + | .................. | + | 0xfffffbfffe800000-0xfffffbffff800000 16M PTE | + | .................. | + | ---[ PCI I/O end ]------------------------------------------------------------------- | + +---------------------------------------------------------------------------------------+ + | ---[ vmemmap start ]----------------------------------------------------------------- | + | .................. | + | 0xfffffc0002000000-0xfffffc0002200000 2M PTE RW NX SHD AF UXN MEM/NORMAL | + | 0xfffffc0002200000-0xfffffc0020000000 478M PTE | + | .................. | + | ---[ vmemmap end ]------------------------------------------------------------------- | + +---------------------------------------------------------------------------------------+ + +``cat /sys/kernel/debug/kernel_page_tables`` output:: + + 0xfff0000001c00000-0xfff0000080000000 2020M PTE RW NX SHD AF UXN MEM/NORMAL-TAGGED + 0xfff0000080000000-0xfff0000800000000 30G PMD + 0xfff0000800000000-0xfff0000800700000 7M PTE RW NX SHD AF UXN MEM/NORMAL-TAGGED + 0xfff0000800700000-0xfff0000800710000 64K PTE ro NX SHD AF UXN MEM/NORMAL-TAGGED + 0xfff0000800710000-0xfff0000880000000 2089920K PTE RW NX SHD AF UXN MEM/NORMAL-TAGGED + 0xfff0000880000000-0xfff0040000000000 4062G PMD + 0xfff0040000000000-0xffff800000000000 3964T PGD -- cgit From 39138093f139c5d03e852c49cc52339543922308 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Fri, 16 Jun 2023 11:31:50 +0100 Subject: arm64: alternatives: make clean_dcache_range_nopatch() noinstr-safe When patching kernel alternatives, we need to be careful not to execute kernel code which is itself subject to patching. In general, if code is executed after the instructions in memory have been patched but prior to the cache maintenance and barriers completing, it could lead to UNPREDICTABLE results. As our regular cache maintenance routines are patched with alternatives, we have a clean_dcache_range_nopatch() function which is *intended* to avoid patchable code and therefore supposed to be safe in the middle of patching alternatives. Unfortunately, it's not marked as 'noinstr', and so can be instrumented with patchable code. Additionally, it calls read_sanitised_ftr_reg() (which may be instrumented with patchable code) to find the sanitized value of CTR_EL0.DminLine, and is therefore not safe to call during patching. Luckily, since commit: 675b0563d6b26aa9 ("arm64: cpufeature: expose arm64_ftr_reg struct for CTR_EL0") ... we can read the sanitised CTR_EL0 value directly, and avoid the call to read_sanitised_ftr_reg(). This patch marks clean_dcache_range_nopatch() as noinstr, and has it read the sanitized CTR_EL0 value directly, avoiding the issues above. As a bonus, this is also an optimization. As read_sanitised_ftr_reg() performs a binary search to find the CTR_EL0 value, reading the value directly avoids this binary search per applied alternative, avoiding some unnecessary work. Signed-off-by: Mark Rutland Cc: Will Deacon Link: https://lore.kernel.org/r/20230616103150.1238132-1-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/kernel/alternative.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c index d32d4ed5519b..9d84bd818f19 100644 --- a/arch/arm64/kernel/alternative.c +++ b/arch/arm64/kernel/alternative.c @@ -121,11 +121,11 @@ static noinstr void patch_alternative(struct alt_instr *alt, * accidentally call into the cache.S code, which is patched by us at * runtime. */ -static void clean_dcache_range_nopatch(u64 start, u64 end) +static noinstr void clean_dcache_range_nopatch(u64 start, u64 end) { u64 cur, d_size, ctr_el0; - ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0); + ctr_el0 = arm64_ftr_reg_ctrel0.sys_val; d_size = 4 << cpuid_feature_extract_unsigned_field(ctr_el0, CTR_EL0_DminLine_SHIFT); cur = start & ~(d_size - 1); -- cgit From 616cb2f4b141852cac3dfffe8354c8bf19e9999d Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Thu, 22 Jun 2023 14:39:45 +0100 Subject: arm64/signal: Restore TPIDR2 register rather than memory state Currently when restoring the TPIDR2 signal context we set the new value from the signal frame in the thread data structure but not the register, following the pattern for the rest of the data we are restoring. This does not work in the case of TPIDR2, the register always has the value for the current task. This means that either we return to userspace and ignore the new value or we context switch and save the register value on top of the newly restored value. Load the value from the signal context into the register instead. Fixes: 39e54499280f ("arm64/signal: Include TPIDR2 in the signal context") Signed-off-by: Mark Brown Cc: # 6.3.x Link: https://lore.kernel.org/r/20230621-arm64-fix-tpidr2-signal-restore-v2-1-c8e8fcc10302@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/kernel/signal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 2cfc810d0a5b..10b407672c42 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -398,7 +398,7 @@ static int restore_tpidr2_context(struct user_ctxs *user) __get_user_error(tpidr2_el0, &user->tpidr2->tpidr2, err); if (!err) - current->thread.tpidr2_el0 = tpidr2_el0; + write_sysreg_s(tpidr2_el0, SYS_TPIDR2_EL0); return err; } -- cgit From f7a5d72edc522b9210521d9b3969eac221eb6ecb Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Thu, 22 Jun 2023 14:39:46 +0100 Subject: kselftest/arm64: Add a test case for TPIDR2 restore Due to the fact that TPIDR2 is intended to be managed by libc we don't currently test modifying it via the signal context since that might disrupt libc's usage of it and cause instability. We can however test the opposite case with less risk, modifying TPIDR2 in a signal handler and making sure that the original value is restored after returning from the signal handler. Add a test which does this. Signed-off-by: Mark Brown Link: https://lore.kernel.org/r/20230621-arm64-fix-tpidr2-signal-restore-v2-2-c8e8fcc10302@kernel.org Signed-off-by: Catalin Marinas --- tools/testing/selftests/arm64/signal/.gitignore | 2 +- .../arm64/signal/testcases/tpidr2_restore.c | 86 ++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/arm64/signal/testcases/tpidr2_restore.c diff --git a/tools/testing/selftests/arm64/signal/.gitignore b/tools/testing/selftests/arm64/signal/.gitignore index 8ab4c86837fd..839e3a252629 100644 --- a/tools/testing/selftests/arm64/signal/.gitignore +++ b/tools/testing/selftests/arm64/signal/.gitignore @@ -4,7 +4,7 @@ fake_sigreturn_* sme_* ssve_* sve_* -tpidr2_siginfo +tpidr2_* za_* zt_* !*.[ch] diff --git a/tools/testing/selftests/arm64/signal/testcases/tpidr2_restore.c b/tools/testing/selftests/arm64/signal/testcases/tpidr2_restore.c new file mode 100644 index 000000000000..f9a86c00c28c --- /dev/null +++ b/tools/testing/selftests/arm64/signal/testcases/tpidr2_restore.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 ARM Limited + * + * Verify that the TPIDR2 register context in signal frames is restored. + */ + +#include +#include +#include +#include +#include +#include + +#include "test_signals_utils.h" +#include "testcases.h" + +#define SYS_TPIDR2 "S3_3_C13_C0_5" + +static uint64_t get_tpidr2(void) +{ + uint64_t val; + + asm volatile ( + "mrs %0, " SYS_TPIDR2 "\n" + : "=r"(val) + : + : "cc"); + + return val; +} + +static void set_tpidr2(uint64_t val) +{ + asm volatile ( + "msr " SYS_TPIDR2 ", %0\n" + : + : "r"(val) + : "cc"); +} + + +static uint64_t initial_tpidr2; + +static bool save_tpidr2(struct tdescr *td) +{ + initial_tpidr2 = get_tpidr2(); + fprintf(stderr, "Initial TPIDR2: %lx\n", initial_tpidr2); + + return true; +} + +static int modify_tpidr2(struct tdescr *td, siginfo_t *si, ucontext_t *uc) +{ + uint64_t my_tpidr2 = get_tpidr2(); + + my_tpidr2++; + fprintf(stderr, "Setting TPIDR2 to %lx\n", my_tpidr2); + set_tpidr2(my_tpidr2); + + return 0; +} + +static void check_tpidr2(struct tdescr *td) +{ + uint64_t tpidr2 = get_tpidr2(); + + td->pass = tpidr2 == initial_tpidr2; + + if (td->pass) + fprintf(stderr, "TPIDR2 restored\n"); + else + fprintf(stderr, "TPIDR2 was %lx but is now %lx\n", + initial_tpidr2, tpidr2); +} + +struct tdescr tde = { + .name = "TPIDR2 restore", + .descr = "Validate that TPIDR2 is restored from the sigframe", + .feats_required = FEAT_SME, + .timeout = 3, + .sig_trig = SIGUSR1, + .init = save_tpidr2, + .run = modify_tpidr2, + .check_result = check_tpidr2, +}; -- cgit